added aio rmq client, added internal queue for master service
This commit is contained in:
nikili0n
@ -1,48 +0,0 @@
|
||||
import os
|
||||
|
||||
from fastapi import APIRouter, Form, HTTPException
|
||||
from starlette.requests import Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.responses import HTMLResponse, FileResponse
|
||||
from starlette.templating import Jinja2Templates
|
||||
|
||||
from src.core.ydl import VideoDownloader
|
||||
from src.exceptions.download_exceptions import SiteNotImplementedException
|
||||
|
||||
main_router = APIRouter()
|
||||
|
||||
templates = Jinja2Templates(directory="templates")
|
||||
|
||||
|
||||
@main_router.get("/", response_class=HTMLResponse)
|
||||
async def index(request: Request):
|
||||
return templates.TemplateResponse("index.html", {"request": request})
|
||||
|
||||
|
||||
@main_router.post('/submit')
|
||||
async def get_url_for_download_video(request: Request, link: str = Form(...)):
|
||||
file_name = VideoDownloader.get_unique_video_filename()
|
||||
ydl_opts = {
|
||||
"forceurl": True,
|
||||
"format": "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
|
||||
"merge_output_format": "mp4",
|
||||
'outtmpl': f'downloads/%(extractor_key)s/{file_name}.%(ext)s',
|
||||
}
|
||||
downloader = VideoDownloader(link=link, ydl_opts=ydl_opts)
|
||||
try:
|
||||
result = downloader.download()
|
||||
link_to_download_video = str(request.base_url) + f"{file_name}.{result['ext']}"
|
||||
except SiteNotImplementedException as ex:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=ex.message
|
||||
)
|
||||
return JSONResponse({"result": link_to_download_video})
|
||||
|
||||
|
||||
@main_router.get('/{file_path}', response_class=FileResponse)
|
||||
async def download_video(file_path):
|
||||
base = os.path.dirname(os.path.dirname(os.path.abspath(file_path)))
|
||||
base_download_dir = os.path.join(base, "src", "downloads")
|
||||
youtube_dir = os.path.join(base_download_dir, "Youtube")
|
||||
return FileResponse(youtube_dir + f'/{file_path}', media_type="video")
|
||||
@ -1,5 +0,0 @@
|
||||
from src.core.ydl import VideoDownloader
|
||||
|
||||
|
||||
def get_downloader(link: str, opts: dict):
|
||||
return VideoDownloader(link=link, ydl_opts=opts)
|
||||
80
src/core/master_service.py
Normal file
80
src/core/master_service.py
Normal file
@ -0,0 +1,80 @@
|
||||
import asyncio
|
||||
import json
|
||||
import concurrent.futures as pool
|
||||
from multiprocessing import freeze_support
|
||||
from collections import deque
|
||||
|
||||
from fastapi import HTTPException
|
||||
from aio_pika import connect
|
||||
from aio_pika.abc import AbstractIncomingMessage
|
||||
|
||||
from src.core.send import body
|
||||
from src.core.ydl import VideoDownloader
|
||||
from src.exceptions.download_exceptions import SiteNotImplementedException
|
||||
|
||||
|
||||
as_queue = asyncio.Queue()
|
||||
|
||||
|
||||
async def on_message(message: AbstractIncomingMessage) -> None:
|
||||
async with message.process():
|
||||
# print(f" [x] Received message {message!r}")
|
||||
await as_queue.put(json.loads(message.body))
|
||||
print(f" Message body is: {message.body!r}")
|
||||
|
||||
|
||||
async def get_messages() -> None:
|
||||
# Perform connection
|
||||
connection = await connect("amqp://guest:guest@localhost/")
|
||||
|
||||
async with connection:
|
||||
# Creating a channel
|
||||
channel = await connection.channel()
|
||||
await channel.set_qos(prefetch_count=1)
|
||||
|
||||
# Declaring queue
|
||||
queue = await channel.declare_queue(
|
||||
"hello",
|
||||
durable=True,
|
||||
)
|
||||
|
||||
# Start listening the queue with name 'task_queue'
|
||||
await queue.consume(on_message)
|
||||
|
||||
print(" [*] Waiting for messages. To exit press CTRL+C")
|
||||
await asyncio.Future()
|
||||
|
||||
|
||||
def get_url_for_download_video(data: str):
|
||||
file_name = VideoDownloader.get_unique_video_filename()
|
||||
ydl_opts = {
|
||||
"format": data["format"],
|
||||
"merge_output_format": data["merge_output_format"],
|
||||
'outtmpl': data["outtmpl"],
|
||||
}
|
||||
downloader = VideoDownloader(link=data["link"], ydl_opts=ydl_opts)
|
||||
try:
|
||||
result = downloader.download()
|
||||
except SiteNotImplementedException as ex:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=ex.message
|
||||
)
|
||||
|
||||
|
||||
async def create_workers(queue):
|
||||
loop = asyncio.get_event_loop()
|
||||
while True:
|
||||
link = await queue.get()
|
||||
with pool.ProcessPoolExecutor(max_workers=8) as executor:
|
||||
tasks = loop.run_in_executor(executor, get_url_for_download_video, link)
|
||||
queue.task_done()
|
||||
|
||||
|
||||
async def main():
|
||||
await asyncio.gather(get_messages(), create_workers(as_queue))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
freeze_support()
|
||||
asyncio.run(main())
|
||||
18
src/main.py
18
src/main.py
@ -1,18 +0,0 @@
|
||||
import uvicorn
|
||||
from fastapi import FastAPI
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from api.api_v1.endpoints.main import main_router
|
||||
|
||||
app = FastAPI(
|
||||
title="PythonProject", openapi_url=f"/api/v1/openapi.json"
|
||||
)
|
||||
|
||||
app.include_router(
|
||||
main_router,
|
||||
)
|
||||
|
||||
app.mount("/downloads", StaticFiles(directory="downloads"), name="downloads")
|
||||
|
||||
if __name__ == '__main__':
|
||||
uvicorn.run("src.main:app", host="localhost", log_level="info")
|
||||
@ -1,115 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Video Downloading</title>
|
||||
</head>
|
||||
<style>
|
||||
.custom-btn {
|
||||
width: 145px;
|
||||
height: 60px;
|
||||
color: #fff;
|
||||
border-radius: 5px;
|
||||
padding: 10px 25px;
|
||||
font-family: 'Lato', sans-serif;
|
||||
font-weight: 500;
|
||||
background: transparent;
|
||||
cursor: pointer;
|
||||
transition: all 0.3s ease;
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
box-shadow:inset 2px 2px 2px 0px rgba(255,255,255,.5),
|
||||
7px 7px 20px 0px rgba(0,0,0,.1),
|
||||
4px 4px 5px 0px rgba(0,0,0,.1);
|
||||
outline: none;
|
||||
}
|
||||
.btn-1 {
|
||||
background: rgb(6,14,131);
|
||||
background: linear-gradient(0deg, rgba(6,14,131,1) 0%, rgba(12,25,180,1) 100%);
|
||||
border: none;
|
||||
}
|
||||
.btn-1:hover {
|
||||
background: rgb(0,3,255);
|
||||
background: linear-gradient(0deg, rgba(0,3,255,1) 0%, rgba(2,126,251,1) 100%);
|
||||
}
|
||||
input {
|
||||
width: 300px;
|
||||
font-size: 13px;
|
||||
padding: 6px 0 4px 10px;
|
||||
border: 1px solid #cecece;
|
||||
background: #F6F6f6;
|
||||
border-radius: 8px;
|
||||
}
|
||||
.col {
|
||||
background: #f0f0f0; /* Цвет фона */
|
||||
width: 1000px; /* Ширина блока */
|
||||
padding: 10px; /* Поля */
|
||||
font-size: 1.5em; /* Размер шрифта */
|
||||
word-wrap: break-word; /* Перенос слов */
|
||||
}
|
||||
@keyframes spinner-border {
|
||||
100% {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
.submit-spinner {
|
||||
display: inline-block;
|
||||
width: 1rem;
|
||||
height: 1rem;
|
||||
vertical-align: -0.125em;
|
||||
border: 0.2em solid currentColor;
|
||||
border-right-color: transparent;
|
||||
border-radius: 50%;
|
||||
-webkit-animation: .75s linear infinite spinner-border;
|
||||
animation: .75s linear infinite spinner-border;
|
||||
}
|
||||
|
||||
.submit-spinner_hide {
|
||||
display: none;
|
||||
}
|
||||
</style>
|
||||
<body>
|
||||
<form method="post" action="/submit" id="download">
|
||||
<input type="text" name="link">
|
||||
<button type="submit" class="custom-btn btn-1"><span class="submit-spinner submit-spinner_hide"></span> Download</button>
|
||||
</form>
|
||||
<div class="col">
|
||||
<p> Ссылка для скачивания:</p>
|
||||
<br>
|
||||
<a id="result" href="" download></a>
|
||||
<div class="loader">
|
||||
<div class="loader_inner"></div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
<script>
|
||||
function sendForm() {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open('POST', document.forms.download.action);
|
||||
xhr.responseType = 'json';
|
||||
xhr.onload = () => {
|
||||
document.forms.download.querySelector('[type="submit"]').disabled = false;
|
||||
document.forms.download.querySelector('.submit-spinner').classList.add('submit-spinner_hide');
|
||||
if (xhr.status !== 200) {
|
||||
return;
|
||||
}
|
||||
const response = xhr.response;
|
||||
result.innerHTML = xhr.response.result;
|
||||
result.href = xhr.response.result;
|
||||
}
|
||||
xhr.onerror = () => {
|
||||
document.forms.download.querySelector('[type="submit"]').disabled = false;
|
||||
document.forms.download.querySelector('.submit-spinner').classList.add('submit-spinner_hide');
|
||||
};
|
||||
document.forms.download.querySelector('[type="submit"]').disabled = true;
|
||||
document.forms.download.querySelector('.submit-spinner').classList.remove('submit-spinner_hide');
|
||||
xhr.send(new FormData(document.forms.download));
|
||||
}
|
||||
// при отправке формы
|
||||
document.forms.download.addEventListener('submit', (e) => {
|
||||
e.preventDefault();
|
||||
sendForm();
|
||||
});
|
||||
</script>
|
||||
</html>
|
||||
Reference in New Issue
Block a user