Dantenerosas/video_downloader_service
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rework redis, rework web for work with array of links

Browse Source
This commit is contained in:
Dantenerosas
2023-09-29 05:53:27 +03:00
committed by nikili0n
parent 1903ec8f5b
commit 314b44aca7
10 changed files with 153 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
src/web/main.py
View File

@ -27,7 +27,6 @@ app.add_middleware(
allow_headers=["*"],
)
'''
await self.app(scope, receive, send)
File "/home/admin/video_downloader_service/.venv/lib/python3.10/site-packages/starlette/routing.py", line 66, in app
@ -61,32 +60,19 @@ queue_name -> {
async def is_task_already_done_or_exist(redis: RedisClient, link: str):
messages = await redis.get_task_done_queue()
temp = [json.loads(msg) for msg in messages]
tasks = [
msg for msg in temp
if msg["link"] == link
and msg["status"] in ["done", "exist"]
]
messages = await redis.get_all_tasks_from_queue(redis.TASKS_DONE_NAME)
tasks = {k.decode("utf-8"): json.loads(v.decode("utf-8")) for k, v in messages.items()} if messages else None
if len(tasks) > 0:
task = tasks[0]
if os.path.exists(os.path.join(os.getcwd(), os.pardir, os.pardir + "/downloads/" + task["result"])):
return task
await redis.del_task_from_task_done_queue(task)
if tasks and link in tasks and tasks[link]["status"] in ["done", "exist"]:
return tasks[link]
async def is_task_in_process(redis: RedisClient, link: str):
messages = await redis.get_tasks()
messages = await redis.get_all_tasks_from_queue(redis.TASKS_NAME)
tasks = {k.decode("utf-8"): json.loads(v.decode("utf-8")) for k, v in messages.items()} if messages else None
tasks = [
literal_eval(message.decode('utf-8')) for message in messages
if literal_eval(message.decode('utf-8'))["link"] == link
]
if len(tasks) > 0:
task = tasks[0]
return task
if tasks and link in tasks:
return tasks[link]
@app.get("/")
@ -101,10 +87,13 @@ async def get_url_for_download_video(request: Request, data: SubmitIn = Depends(
# TODO: где-то не обновился статус после выполнения\провала задачи
task_in_process = await is_task_in_process(red, data.link)
if task_in_process:
return JSONResponse({"result": "Задача в работе. Ожидайте"})
return JSONResponse(status_code=202, content={"result": "Задача в работе. Ожидайте"})
if task_done:
link_to_download_video = str(request.base_url) + "get/?file_path=" + task_done["result"]
return JSONResponse({"result": link_to_download_video})
if isinstance(task_done["result"], str):
links_to_download_video = [str(request.base_url) + "get/?file_path=" + task_done["result"]]
else:
links_to_download_video = [str(request.base_url) + "get/?file_path=" + path for path in task_done["result"]]
return JSONResponse({"result": links_to_download_video})
# TODO: учесть, что если делать запрос CURL\urllib3\etc, в теле может быть несколько ссылок -> должно быть создано несколько задач
async with await connect("amqp://guest:guest@localhost/") as connection:
@ -115,7 +104,7 @@ async def get_url_for_download_video(request: Request, data: SubmitIn = Depends(
"link": data.link,
"format": f"bestvideo[ext={data.video_format.value}]+bestaudio[ext={data.audio_format.value}]/best[ext={data.video_format.value}]/best",
"merge_output_format": data.merge_output_format.value,
"outtmpl": f"downloads/%(extractor_key)s/%(id)s_%(width)sp.%(ext)s",
"outtmpl": f"downloads/%(extractor_key)s/%(id)s_%(resolution)s.%(ext)s",
}, ]
# Sending the message
for link in body:
@ -147,44 +136,41 @@ async def download_video(file_path):
with open(base_download_dir + f'/{file_path}', mode="rb") as file_like:
yield from file_like
return StreamingResponse(iterfile(), headers={'Content-Disposition': f'inline; filename="{file_path}"'}, media_type="video")
return StreamingResponse(iterfile(), headers={'Content-Disposition': f'inline; filename="{file_path}"'},
media_type="video")
@app.post('/check/', response_class=FileResponse, status_code=200)
async def download_video(data: CheckIn, request: Request):
try:
red = RedisClient()
messages_task_done = await red.get_all_tasks_from_queue(red.TASKS_DONE_NAME)
messages_tasks = await red.get_all_tasks_from_queue(red.TASKS_NAME)
messages_task_done = await red.get_task_done_queue()
messages_tasks = await red.get_tasks()
tasks_done = {k.decode("utf-8"): json.loads(v.decode("utf-8")) for k, v in
messages_task_done.items()} if messages_task_done else None
tasks_done = [
literal_eval(message.decode('utf-8')) for message in messages_task_done
if literal_eval(message.decode('utf-8'))["link"] == data.link
]
tasks = [
literal_eval(message.decode('utf-8')) for message in messages_tasks
if literal_eval(message.decode('utf-8'))["link"] == data.link
]
tasks = {k.decode("utf-8"): json.loads(v.decode("utf-8")) for k, v in
messages_tasks.items()} if messages_tasks else None
error_tasks = [
tasks_done.pop(tasks_done.index(error_task)) for error_task in tasks_done if error_task["status"] == "error"
] if tasks_done else None
if tasks and len(tasks) > 0:
task = tasks[0]
if tasks and data.link in tasks:
return JSONResponse(
status_code=202,
content={"result": f"Задача {task['link']} в данный момент в работе, выполняется"}
content={"result": f"Задача {data.link} в данный момент в работе, выполняется"}
)
# TODO: если уже была попытка сделать задачу и в редисе она с ошибкой, то переташить её в очередь на выполнение с очисткой состояние об ошибке
if error_tasks and len(error_tasks) > 0:
error_task = error_tasks[0]
await red.del_task_from_task_done_queue(error_task)
if data.link in tasks_done and tasks_done[data.link]["status"] == "error":
await red.del_task_from_task_done_queue(tasks_done[data.link])
return JSONResponse(status_code=510,
content={"result": f"Задача выполнена с ошибкой, попробуйте загрузить еще раз"})
if len(tasks_done) > 0:
link_to_download_video = str(request.base_url) + "get/?file_path=" + tasks_done[0]["result"]
return JSONResponse({"result": link_to_download_video})
if tasks_done and data.link in tasks_done:
if isinstance(tasks_done[data.link]["result"], str):
links_to_download_video = [str(request.base_url) + "get/?file_path=" + tasks_done[data.link]["result"]]
else:
links_to_download_video = [str(request.base_url) + "get/?file_path=" + path for path in
tasks_done[data.link]["result"]]
return JSONResponse({"result": links_to_download_video})
return JSONResponse(status_code=404, content={"result": "Задача не найдена"})
except (AttributeError, IndexError):
return JSONResponse(status_code=404, content={"result": "Задача не найдена"})

54
src/web/templates/index.html
View File

@ -77,7 +77,7 @@
<input type="text" name="merge_output_format" placeholder="merge_output_format">
<button type="submit" class="custom-btn btn-1"><span class="submit-spinner submit-spinner_hide"></span> Download</button>
</form>
<div class="col">
<div id="linksList" class="col">
<p> Ссылка для скачивания:</p>
<br>
<a id="result" href="" download></a>
@ -87,12 +87,32 @@
</div>
</body>
<script>
function createLink(url) {
var link = document.createElement('a');
link.className = "addedLink";
link.setAttribute('download', '');
link.innerHTML = url;
link.href = url;
return link;
}
function createLinksList(urls) {
var mydiv = document.getElementById("linksList");
urls.forEach((element) => {
mydiv.appendChild(createLink(element));
var br = document.createElement('br');
br.className = "addedBr";
mydiv.appendChild(br);
}
);
};
function sendReq() {
document.forms.download.querySelector('[type="submit"]').disabled = true;
document.forms.download.querySelector('.submit-spinner').classList.remove('submit-spinner_hide');
const link = document.getElementById("link").value
const xhr2 = new XMLHttpRequest();
// TODO: скорректировать ссылку, она должна быть относительной
xhr2.open('POST', 'http://'+document.location.host+'/check/');
xhr2.responseType = 'json';
xhr2.setRequestHeader("Content-Type", "application/json;charset=UTF-8");
@ -113,8 +133,11 @@
document.forms.download.querySelector('.submit-spinner').classList.add('submit-spinner_hide');
};
} else if (xhr2.status === 200) {
result.innerHTML = xhr2.response.result;
result.href = xhr2.response.result;
console.log(xhr2.response)
if (document.getElementById("result").texContent !== '') {
document.getElementById("result").textContent = ''
};
createLinksList(xhr2.response["result"]);
document.forms.download.querySelector('[type="submit"]').disabled = false;
document.forms.download.querySelector('.submit-spinner').classList.add('submit-spinner_hide');
};
@ -127,6 +150,16 @@
function sendForm() {
const links = document.querySelectorAll('.addedLink');
links.forEach(link => {
link.remove();
}
);
const brs = document.querySelectorAll('.addedBr');
brs.forEach(br => {
br.remove();
}
);
const xhr = new XMLHttpRequest();
xhr.open('POST', document.forms.download.action);
xhr.responseType = 'json';
@ -140,12 +173,17 @@
};
const response = xhr.response;
result.innerHTML = xhr.response.result;
result.href = xhr.response.result;
console.log(response);
if (xhr.status === 201) {
sendReq()
}
} else if (xhr.status === 202) {
result.innerHTML = xhr.response.result;
result.href = xhr.response.result;
} else {
console.log(response["result"]);
createLinksList(response["result"]);
};
}
xhr.onerror = () => {
document.forms.download.querySelector('[type="submit"]').disabled = false;