Dantenerosas/video_downloader_service
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

minor fixes, added web serer

Browse Source
This commit is contained in:
nikili0n
2023-09-20 14:43:59 +03:00
parent aa519d622f
commit 9b7e006eec
8 changed files with 252 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/core/master_service.py
View File

@@ -48,7 +48,7 @@ class MasterService:
)
result = await download_task
await redis.del_task_from_tasks_and_add_to_task_done(task=video_params)
await redis.del_task_from_tasks_and_add_to_task_done(task={"link": video_params["link"], "result": result})
# TODO process result
self.queue.task_done()
@@ -76,8 +76,8 @@ class MasterService:
@staticmethod
def video_processing_executor(video_params: dict):
try:
MasterService.video_download(video_params=video_params)
return Result(result_type=ResultTypeEnum.DONE)
result = MasterService.video_download(video_params=video_params)
return result
except Exception as ex:
return Result(result_type=ResultTypeEnum.EXCEPTION, value=ex)
# TODO upload to server

20
src/core/redis_client.py
View File

@@ -18,8 +18,8 @@ class RedisClient:
async def _set_task_done(self, task: dict) -> int:
async with self.connection as connection:
res = await connection.set(
f'{self.TASKS_DONE_NAME}:1:{task["link"]}',
res = await connection.sadd(
f'{self.TASKS_DONE_NAME}:1',
json.dumps(task, indent=4).encode('utf-8')
)
return res
@@ -53,3 +53,19 @@ class RedisClient:
await self._del_task(task)
return await self._set_task_done(task)
async def get_task_done_queue(self) -> set:
async with self.connection as connection:
res = await connection.smembers(self.TASKS_DONE_NAME + f":1")
return res
async def del_task_from_task_done_queue(self, task) -> int:
async with self.connection as connection:
res = await connection.srem(self.TASKS_DONE_NAME + f":1", json.dumps(task, indent=4).encode('utf-8'))
return res
async def get_tasks_queue(self) -> set:
async with self.connection as connection:
res = await connection.json().get(self.TASKS_NAME)
return res

6
src/core/uploader.py
View File

@@ -15,7 +15,7 @@ def main():
if not found:
client.make_bucket("clean-internet-oculus-integration-dev")
else:
print("Bucket 'asiatrip' already exists")
print("Bucket 'clean-internet-oculus-integration-dev' already exists")
# Upload '/home/user/Photos/asiaphotos.zip' as object name
# 'asiaphotos-2015.zip' to bucket 'asiatrip'.
@@ -23,8 +23,8 @@ def main():
"clean-internet-oculus-integration-dev", "4uv2GNc_ybc_1080p.mp4", "/Users/garickbadalov/PycharmProjects/video_downloader_service/downloads/Youtube/4uv2GNc_ybc_1080p.mp4",
)
print(
"'/home/user/Photos/asiaphotos.zip' is successfully uploaded as "
"object 'asiaphotos-2015.zip' to bucket 'asiatrip'."
"'/Users/garickbadalov/PycharmProjects/video_downloader_service/downloads/Youtube/4uv2GNc_ybc_1080p.mp4' is successfully uploaded as "
"object '4uv2GNc_ybc_1080p.mp4' to bucket 'clean-internet-oculus-integration-dev'."
)

20
src/parsers/MyMail/my_mail_parser.py
View File

@@ -1,7 +1,7 @@
import os
import uuid
import requests
from http.cookies import SimpleCookie
from playwright.sync_api import Playwright
from playwright.sync_api import sync_playwright
@@ -19,25 +19,23 @@ class MyMailParser(BaseParser):
page = context.new_page()
mobile_url = f"{self.params['link'][0:8]}m.{self.params['link'][8:]}"
page.goto(url=mobile_url)
cc = context.cookies()
cookies = {cookie["name"]: cookie["value"] for cookie in cc}
link = page.get_attribute("xpath=//video", "src")
link = "https:" + link
title = page.locator("xpath=//div[@class='event-text__title']").text_content()
return link, title
title = cookies["video_key"]
return link, title, cookies
def video_download(self, link: str = None, title: str = None):
if not link and not title:
with sync_playwright() as playwright:
link, title = self.get_video_link(playwright)
link, title, cookies = self.get_video_link(playwright)
if os.path.exists(os.path.join(os.getcwd() + f"MyMailRu/{title}.mp4")):
return Result(result_type=ResultTypeEnum.EXIST)
rawdata = "searchuid=4515257701686610918; p=ki8AAAYkJdcA; act=064d11655c924c9f8f2aad0181a06a4b; o=:1763:AUAQ.m; oid=22SgCdFE5g2ZEFHy1FkYW; mrcu=5A5B64F228CC3AC265485FC5AE55; re_theme=system; re_theme_actual=dark; s=fver=0|rt=1|dpr=2|ww=1728|wh=963; ph_tp_horo-mail-ru=t=1|d=1693591768453; tmr_lvid=26ef811c203f1c0c0e5d1c8af1a4671b; tmr_lvidTS=1693591768481; _ym_uid=1693591769619458564; _ym_d=1693591769; ph_v_my-mail-ru=1; mrhc=CB75tAx8UrwCaiqE85YXWoCM2+CTT6/VsTcMdxv4iCM=; mr_my_b=1; _ga=GA1.2.2000378679.1694259228; mtrc=%7B%22mytrackerid%22%3A52867%2C%22tmr_lvid%22%3A%2226ef811c203f1c0c0e5d1c8af1a4671b%22%7D; c=FuoAZQEAsHsTAAAUAQgACQAAgLrElILY4CDYNvMTASDQrSUa; b=nUwAAJBoPmMDosXR5CCG5oQO4ltqxSBq54QOYpLl6yBiWW0VIvzl6zAOfNwNOtuHt6ADAAAIpgR06GAGp1YMpgB06AAA; i=AQAQDgNlCQATAAguDyABAYwDApADARgHAewHATwJAUMLARkUAXseAjAgAfUgAfYgAfcgAfggAfEiAZMCCHYnbgABAQIBAgIBBwIBCAIBCQIBDgIBDwIBEQIBEgIBFwIBGAIBUQUBVgUBaAUBdAUBdQUBoAUBoQUBpAUBpgUBqQUBegYBDgsBKQsBLgsBxQsBxwsByQsBzAsBzQsBcA0BdQ0BeA0BvQ0B6BAB6RAB6hABw2MB3AQIBAEBAAHhBAkBAeIECgQGB80HOgUIDQQqAgEACAELCAEeEAHWBggEAQEAAb0HCAQBoxUBiQ0FAgHz; video_key=192bed9054db7a4efa7943ad834c7a2e05a55237; VID=0eXAI6071-IK00000t1kP4oK:::0-0-a1b105a-9aaf457:CAASEL33YAsZEz357mCA71F8QJgacM9HfhwzMJ-j3X3e-iJIE0DIiLWfRhfTc3GgyUNfH8_EwadLkVinwp0LA-QyaRe9p0A_ZR0y1i9Hk8aVl8Q8ZB_Qd_hCZN_SfHmeOvHeoe6QBCvz5w2SHcI2iFuAXKJkJMvNuYwSeBLdWhCXvsK5M_M"
cookie = SimpleCookie()
cookie.load(rawdata)
cookies = {k: v.value for k, v in cookie.items()}
if os.path.exists(os.path.join(os.getcwd() + f"/downloads/MyMailRu/{title}.mp4")):
return f"MyMailRu/{title}.mp4"#Result(result_type=ResultTypeEnum.EXIST)
self.make_sure_path_exists()
video_response = requests.get(link, cookies=cookies)
with open(self.BASE_DIR + f"/{title}.mp4", "wb") as output:
output.write(video_response.content)
return f"MyMailRu/{title}.mp4"

8
src/parsers/Yappy/yappy_parser.py
View File

@@ -1,4 +1,5 @@
import os
import uuid
import requests
@@ -17,17 +18,18 @@ class YappyParser(BaseParser):
soup = BeautifulSoup(resp.text, 'lxml')
link = soup.find('video').get("src")
title = soup.find('title').get_text()
title = soup.find('video').get("id")
return link, title
def video_download(self, link: str = None, title: str = None):
if not link and not title:
link, title = self.get_video_link()
if os.path.exists(os.path.join(os.getcwd() + f"Yappy/{title}.mp4")):
return Result(result_type=ResultTypeEnum.EXIST)
if os.path.exists(os.path.join(os.getcwd() + f"/downloads/Yappy/{title}.mp4")):
return f"Yappy/{title}.mp4"
video_response = requests.get(link)
self.make_sure_path_exists()
with open(self.BASE_DIR + f"/{title}.mp4", "wb") as output:
output.write(video_response.content)
return f"Yappy/{title}.mp4"

3
src/parsers/base_parser.py
View File

@@ -24,6 +24,7 @@ class BaseParser:
}
downloader = VideoDownloader(link=self.params["link"], ydl_opts=ydl_opts)
video_info = downloader.get_info()
#TODO Добавить динамеческое имя директории сервиса для проверки дублирования
if os.path.exists(
os.path.join(os.getcwd() + f"Youtube/{video_info['id']}_{video_info['width']}.{video_info['ext']}")
):
@@ -31,7 +32,7 @@ class BaseParser:
try:
downloader.ydl_opts["quiet"] = False
result = downloader.download()
return result
return f"{video_info['extractor_key']}/{result['id']}_{result['width']}p.{result['ext']}"
except SiteNotImplementedException as ex:
raise HTTPException(
status_code=400,

97
src/web/main.py Normal file
View File

@@ -0,0 +1,97 @@
import asyncio
import json
import os
from ast import literal_eval
import uvicorn
from aio_pika import connect, Message, DeliveryMode
from fastapi import FastAPI, Request, Form, HTTPException
from starlette.middleware.cors import CORSMiddleware
from starlette.responses import JSONResponse, FileResponse, StreamingResponse
from starlette.templating import Jinja2Templates
from src.core.redis_client import RedisClient
app = FastAPI(
title="video_downloader", openapi_url=f"/api/v1/openapi.json"
)
templates = Jinja2Templates(directory="templates")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
@app.get("/")
async def index(request: Request):
return templates.TemplateResponse("index.html", {"request": request})
@app.post('/submit/')
async def get_url_for_download_video(request: Request, link: str = Form(...)):
connection = await connect("amqp://guest:guest@localhost/")
async with connection:
# Creating a channel
channel = await connection.channel()
body = [
{
"link": link,
"parser": "base",
"format": "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
"merge_output_format": "mp4",
"outtmpl": f"downloads/%(extractor_key)s/%(id)s_%(width)sp.%(ext)s",
}, ]
# Sending the message
for link in body:
if "mail" in link["link"]:
link["parser"] = "MyMailRu"
elif "yappy" in link["link"]:
link["parser"] = "Yappy"
message = Message(
json.dumps(link, indent=4).encode('utf-8'), delivery_mode=DeliveryMode.PERSISTENT,
)
await channel.default_exchange.publish(
message,
routing_key='hello',
)
print(f" [x] Sent '{link}'")
red = RedisClient()
while True:
try:
mes = await red.get_task_done_queue()
task = literal_eval(list(mes)[0].decode('utf-8'))
if task["link"] == link["link"]:
await red.del_task_from_task_done_queue(task)
break
await asyncio.sleep(5)
except (AttributeError, IndexError):
await asyncio.sleep(5)
continue
link_to_download_video = str(request.base_url) + "get/?file_path=" + task["result"]
return JSONResponse({"result": link_to_download_video})
@app.get('/get/', response_class=FileResponse, status_code=200)
async def download_video(file_path):
base = os.path.dirname(os.path.dirname(os.path.abspath(file_path)))
base_download_dir = os.path.join(base, os.pardir, os.pardir, "downloads")
def iterfile():
with open(base_download_dir + f'/{file_path}', mode="rb") as file_like:
yield from file_like
return StreamingResponse(iterfile(), media_type="video/mp4")
if __name__ == '__main__':
uvicorn.run("src.web.main:app", host="localhost", log_level="info")

115
src/web/templates/index.html Normal file
View File

@@ -0,0 +1,115 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Video Downloading</title>
</head>
<style>
.custom-btn {
width: 145px;
height: 60px;
color: #fff;
border-radius: 5px;
padding: 10px 25px;
font-family: 'Lato', sans-serif;
font-weight: 500;
background: transparent;
cursor: pointer;
transition: all 0.3s ease;
position: relative;
display: inline-block;
box-shadow:inset 2px 2px 2px 0px rgba(255,255,255,.5),
7px 7px 20px 0px rgba(0,0,0,.1),
4px 4px 5px 0px rgba(0,0,0,.1);
outline: none;
}
.btn-1 {
background: rgb(6,14,131);
background: linear-gradient(0deg, rgba(6,14,131,1) 0%, rgba(12,25,180,1) 100%);
border: none;
}
.btn-1:hover {
background: rgb(0,3,255);
background: linear-gradient(0deg, rgba(0,3,255,1) 0%, rgba(2,126,251,1) 100%);
}
input {
width: 300px;
font-size: 13px;
padding: 6px 0 4px 10px;
border: 1px solid #cecece;
background: #F6F6f6;
border-radius: 8px;
}
.col {
background: #f0f0f0; /* Цвет фона */
width: 1000px; /* Ширина блока */
padding: 10px; /* Поля */
font-size: 1.5em; /* Размер шрифта */
word-wrap: break-word; /* Перенос слов */
}
@keyframes spinner-border {
100% {
transform: rotate(360deg);
}
}
.submit-spinner {
display: inline-block;
width: 1rem;
height: 1rem;
vertical-align: -0.125em;
border: 0.2em solid currentColor;
border-right-color: transparent;
border-radius: 50%;
-webkit-animation: .75s linear infinite spinner-border;
animation: .75s linear infinite spinner-border;
}
.submit-spinner_hide {
display: none;
}
</style>
<body>
<form method="post" action="/submit" id="download">
<input type="text" name="link">
<button type="submit" class="custom-btn btn-1"><span class="submit-spinner submit-spinner_hide"></span> Download</button>
</form>
<div class="col">
<p> Ссылка для скачивания:</p>
<br>
<a id="result" href="" download></a>
<div class="loader">
<div class="loader_inner"></div>
</div>
</div>
</body>
<script>
function sendForm() {
const xhr = new XMLHttpRequest();
xhr.open('POST', document.forms.download.action);
xhr.responseType = 'json';
xhr.onload = () => {
document.forms.download.querySelector('[type="submit"]').disabled = false;
document.forms.download.querySelector('.submit-spinner').classList.add('submit-spinner_hide');
if (xhr.status !== 200) {
return;
}
const response = xhr.response;
result.innerHTML = xhr.response.result;
result.href = xhr.response.result;
}
xhr.onerror = () => {
document.forms.download.querySelector('[type="submit"]').disabled = false;
document.forms.download.querySelector('.submit-spinner').classList.add('submit-spinner_hide');
};
document.forms.download.querySelector('[type="submit"]').disabled = true;
document.forms.download.querySelector('.submit-spinner').classList.remove('submit-spinner_hide');
xhr.send(new FormData(document.forms.download));
}
// при отправке формы
document.forms.download.addEventListener('submit', (e) => {
e.preventDefault();
sendForm();
});
</script>
</html>