minor fixes, added web serer

src/core/master_service.py

@@ -48,7 +48,7 @@ class MasterService:
             )
 
             result = await download_task
-            await redis.del_task_from_tasks_and_add_to_task_done(task=video_params)
+            await redis.del_task_from_tasks_and_add_to_task_done(task={"link": video_params["link"], "result": result})
             # TODO process result
             self.queue.task_done()
 
@@ -76,8 +76,8 @@ class MasterService:
     @staticmethod
     def video_processing_executor(video_params: dict):
         try:
-            MasterService.video_download(video_params=video_params)
+            result = MasterService.video_download(video_params=video_params)
-            return Result(result_type=ResultTypeEnum.DONE)
+            return result
         except Exception as ex:
             return Result(result_type=ResultTypeEnum.EXCEPTION, value=ex)
         # TODO upload to server

src/core/redis_client.py

@@ -18,8 +18,8 @@ class RedisClient:
 
     async def _set_task_done(self, task: dict) -> int:
         async with self.connection as connection:
-            res = await connection.set(
+            res = await connection.sadd(
-                f'{self.TASKS_DONE_NAME}:1:{task["link"]}',
+                f'{self.TASKS_DONE_NAME}:1',
                 json.dumps(task, indent=4).encode('utf-8')
                                        )
         return res
@@ -53,3 +53,19 @@ class RedisClient:
         await self._del_task(task)
         return await self._set_task_done(task)
 
+    async def get_task_done_queue(self) -> set:
+        async with self.connection as connection:
+            res = await connection.smembers(self.TASKS_DONE_NAME + f":1")
+        return res
+
+    async def del_task_from_task_done_queue(self, task) -> int:
+        async with self.connection as connection:
+            res = await connection.srem(self.TASKS_DONE_NAME + f":1", json.dumps(task, indent=4).encode('utf-8'))
+        return res
+
+    async def get_tasks_queue(self) -> set:
+        async with self.connection as connection:
+            res = await connection.json().get(self.TASKS_NAME)
+        return res
+
+

src/core/uploader.py

@@ -15,7 +15,7 @@ def main():
     if not found:
         client.make_bucket("clean-internet-oculus-integration-dev")
     else:
-        print("Bucket 'asiatrip' already exists")
+        print("Bucket 'clean-internet-oculus-integration-dev' already exists")
 
     # Upload '/home/user/Photos/asiaphotos.zip' as object name
     # 'asiaphotos-2015.zip' to bucket 'asiatrip'.
@@ -23,8 +23,8 @@ def main():
         "clean-internet-oculus-integration-dev", "4uv2GNc_ybc_1080p.mp4", "/Users/garickbadalov/PycharmProjects/video_downloader_service/downloads/Youtube/4uv2GNc_ybc_1080p.mp4",
     )
     print(
-        "'/home/user/Photos/asiaphotos.zip' is successfully uploaded as "
+        "'/Users/garickbadalov/PycharmProjects/video_downloader_service/downloads/Youtube/4uv2GNc_ybc_1080p.mp4' is successfully uploaded as "
-        "object 'asiaphotos-2015.zip' to bucket 'asiatrip'."
+        "object '4uv2GNc_ybc_1080p.mp4' to bucket 'clean-internet-oculus-integration-dev'."
     )
 
 

src/parsers/MyMail/my_mail_parser.py

@@ -1,7 +1,7 @@
 import os
+import uuid
 
 import requests
-from http.cookies import SimpleCookie
 
 from playwright.sync_api import Playwright
 from playwright.sync_api import sync_playwright
@@ -19,25 +19,23 @@ class MyMailParser(BaseParser):
         page = context.new_page()
         mobile_url = f"{self.params['link'][0:8]}m.{self.params['link'][8:]}"
         page.goto(url=mobile_url)
+        cc = context.cookies()
+        cookies = {cookie["name"]: cookie["value"] for cookie in cc}
         link = page.get_attribute("xpath=//video", "src")
         link = "https:" + link
-        title = page.locator("xpath=//div[@class='event-text__title']").text_content()
+        title = cookies["video_key"]
-        return link, title
+        return link, title, cookies
 
     def video_download(self, link: str = None, title: str = None):
         if not link and not title:
             with sync_playwright() as playwright:
-                link, title = self.get_video_link(playwright)
+                link, title, cookies = self.get_video_link(playwright)
 
-        if os.path.exists(os.path.join(os.getcwd() + f"MyMailRu/{title}.mp4")):
+        if os.path.exists(os.path.join(os.getcwd() + f"/downloads/MyMailRu/{title}.mp4")):
-            return Result(result_type=ResultTypeEnum.EXIST)
+            return f"MyMailRu/{title}.mp4"#Result(result_type=ResultTypeEnum.EXIST)
-
-        rawdata = "searchuid=4515257701686610918; p=ki8AAAYkJdcA; act=064d11655c924c9f8f2aad0181a06a4b; o=:1763:AUAQ.m; oid=22SgCdFE5g2ZEFHy1FkYW; mrcu=5A5B64F228CC3AC265485FC5AE55; re_theme=system; re_theme_actual=dark; s=fver=0|rt=1|dpr=2|ww=1728|wh=963; ph_tp_horo-mail-ru=t=1|d=1693591768453; tmr_lvid=26ef811c203f1c0c0e5d1c8af1a4671b; tmr_lvidTS=1693591768481; _ym_uid=1693591769619458564; _ym_d=1693591769; ph_v_my-mail-ru=1; mrhc=CB75tAx8UrwCaiqE85YXWoCM2+CTT6/VsTcMdxv4iCM=; mr_my_b=1; _ga=GA1.2.2000378679.1694259228; mtrc=%7B%22mytrackerid%22%3A52867%2C%22tmr_lvid%22%3A%2226ef811c203f1c0c0e5d1c8af1a4671b%22%7D; c=FuoAZQEAsHsTAAAUAQgACQAAgLrElILY4CDYNvMTASDQrSUa; b=nUwAAJBoPmMDosXR5CCG5oQO4ltqxSBq54QOYpLl6yBiWW0VIvzl6zAOfNwNOtuHt6ADAAAIpgR06GAGp1YMpgB06AAA; i=AQAQDgNlCQATAAguDyABAYwDApADARgHAewHATwJAUMLARkUAXseAjAgAfUgAfYgAfcgAfggAfEiAZMCCHYnbgABAQIBAgIBBwIBCAIBCQIBDgIBDwIBEQIBEgIBFwIBGAIBUQUBVgUBaAUBdAUBdQUBoAUBoQUBpAUBpgUBqQUBegYBDgsBKQsBLgsBxQsBxwsByQsBzAsBzQsBcA0BdQ0BeA0BvQ0B6BAB6RAB6hABw2MB3AQIBAEBAAHhBAkBAeIECgQGB80HOgUIDQQqAgEACAELCAEeEAHWBggEAQEAAb0HCAQBoxUBiQ0FAgHz; video_key=192bed9054db7a4efa7943ad834c7a2e05a55237; VID=0eXAI6071-IK00000t1kP4oK:::0-0-a1b105a-9aaf457:CAASEL33YAsZEz357mCA71F8QJgacM9HfhwzMJ-j3X3e-iJIE0DIiLWfRhfTc3GgyUNfH8_EwadLkVinwp0LA-QyaRe9p0A_ZR0y1i9Hk8aVl8Q8ZB_Qd_hCZN_SfHmeOvHeoe6QBCvz5w2SHcI2iFuAXKJkJMvNuYwSeBLdWhCXvsK5M_M"
-        cookie = SimpleCookie()
-        cookie.load(rawdata)
-        cookies = {k: v.value for k, v in cookie.items()}
 
         self.make_sure_path_exists()
         video_response = requests.get(link, cookies=cookies)
         with open(self.BASE_DIR + f"/{title}.mp4", "wb") as output:
             output.write(video_response.content)
+        return f"MyMailRu/{title}.mp4"

src/parsers/Yappy/yappy_parser.py

@@ -1,4 +1,5 @@
 import os
+import uuid
 
 import requests
 
@@ -17,17 +18,18 @@ class YappyParser(BaseParser):
         soup = BeautifulSoup(resp.text, 'lxml')
 
         link = soup.find('video').get("src")
-        title = soup.find('title').get_text()
+        title = soup.find('video').get("id")
         return link, title
 
     def video_download(self, link: str = None, title: str = None):
         if not link and not title:
             link, title = self.get_video_link()
 
-        if os.path.exists(os.path.join(os.getcwd() + f"Yappy/{title}.mp4")):
+        if os.path.exists(os.path.join(os.getcwd() + f"/downloads/Yappy/{title}.mp4")):
-            return Result(result_type=ResultTypeEnum.EXIST)
+            return f"Yappy/{title}.mp4"
 
         video_response = requests.get(link)
         self.make_sure_path_exists()
         with open(self.BASE_DIR + f"/{title}.mp4", "wb") as output:
             output.write(video_response.content)
+        return f"Yappy/{title}.mp4"

src/parsers/base_parser.py

@@ -24,6 +24,7 @@ class BaseParser:
         }
         downloader = VideoDownloader(link=self.params["link"], ydl_opts=ydl_opts)
         video_info = downloader.get_info()
+        #TODO Добавить динамеческое имя директории сервиса для проверки дублирования
         if os.path.exists(
                 os.path.join(os.getcwd() + f"Youtube/{video_info['id']}_{video_info['width']}.{video_info['ext']}")
         ):
@@ -31,7 +32,7 @@ class BaseParser:
         try:
             downloader.ydl_opts["quiet"] = False
             result = downloader.download()
-            return result
+            return f"{video_info['extractor_key']}/{result['id']}_{result['width']}p.{result['ext']}"
         except SiteNotImplementedException as ex:
             raise HTTPException(
                 status_code=400,

src/web/main.py

@@ -0,0 +1,97 @@
+import asyncio
+import json
+import os
+from ast import literal_eval
+
+import uvicorn
+from aio_pika import connect, Message, DeliveryMode
+from fastapi import FastAPI, Request, Form, HTTPException
+from starlette.middleware.cors import CORSMiddleware
+from starlette.responses import JSONResponse, FileResponse, StreamingResponse
+from starlette.templating import Jinja2Templates
+
+from src.core.redis_client import RedisClient
+
+app = FastAPI(
+    title="video_downloader", openapi_url=f"/api/v1/openapi.json"
+)
+
+templates = Jinja2Templates(directory="templates")
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get("/")
+async def index(request: Request):
+    return templates.TemplateResponse("index.html", {"request": request})
+
+
+@app.post('/submit/')
+async def get_url_for_download_video(request: Request, link: str = Form(...)):
+    connection = await connect("amqp://guest:guest@localhost/")
+
+    async with connection:
+        # Creating a channel
+        channel = await connection.channel()
+        body = [
+
+            {
+                "link": link,
+                "parser": "base",
+                "format": "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
+                "merge_output_format": "mp4",
+                "outtmpl": f"downloads/%(extractor_key)s/%(id)s_%(width)sp.%(ext)s",
+            }, ]
+        # Sending the message
+        for link in body:
+            if "mail" in link["link"]:
+                link["parser"] = "MyMailRu"
+            elif "yappy" in link["link"]:
+                link["parser"] = "Yappy"
+            message = Message(
+                json.dumps(link, indent=4).encode('utf-8'), delivery_mode=DeliveryMode.PERSISTENT,
+            )
+            await channel.default_exchange.publish(
+                message,
+                routing_key='hello',
+            )
+
+        print(f" [x] Sent '{link}'")
+        red = RedisClient()
+
+        while True:
+            try:
+                mes = await red.get_task_done_queue()
+                task = literal_eval(list(mes)[0].decode('utf-8'))
+                if task["link"] == link["link"]:
+                    await red.del_task_from_task_done_queue(task)
+                    break
+                await asyncio.sleep(5)
+            except (AttributeError, IndexError):
+                await asyncio.sleep(5)
+                continue
+        link_to_download_video = str(request.base_url) + "get/?file_path=" + task["result"]
+
+    return JSONResponse({"result": link_to_download_video})
+
+
+@app.get('/get/', response_class=FileResponse, status_code=200)
+async def download_video(file_path):
+    base = os.path.dirname(os.path.dirname(os.path.abspath(file_path)))
+    base_download_dir = os.path.join(base, os.pardir, os.pardir, "downloads")
+
+    def iterfile():
+        with open(base_download_dir + f'/{file_path}', mode="rb") as file_like:
+            yield from file_like
+
+    return StreamingResponse(iterfile(), media_type="video/mp4")
+
+
+if __name__ == '__main__':
+    uvicorn.run("src.web.main:app", host="localhost", log_level="info")

src/web/templates/index.html

@@ -0,0 +1,115 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Video Downloading</title>
+</head>
+<style>
+    .custom-btn {
+      width: 145px;
+      height: 60px;
+      color: #fff;
+      border-radius: 5px;
+      padding: 10px 25px;
+      font-family: 'Lato', sans-serif;
+      font-weight: 500;
+      background: transparent;
+      cursor: pointer;
+      transition: all 0.3s ease;
+      position: relative;
+      display: inline-block;
+       box-shadow:inset 2px 2px 2px 0px rgba(255,255,255,.5),
+       7px 7px 20px 0px rgba(0,0,0,.1),
+       4px 4px 5px 0px rgba(0,0,0,.1);
+      outline: none;
+    }
+    .btn-1 {
+      background: rgb(6,14,131);
+      background: linear-gradient(0deg, rgba(6,14,131,1) 0%, rgba(12,25,180,1) 100%);
+      border: none;
+    }
+    .btn-1:hover {
+       background: rgb(0,3,255);
+    background: linear-gradient(0deg, rgba(0,3,255,1) 0%, rgba(2,126,251,1) 100%);
+    }
+    input {
+	width: 300px;
+	font-size: 13px;
+	padding: 6px 0 4px 10px;
+	border: 1px solid #cecece;
+	background: #F6F6f6;
+	border-radius: 8px;
+    }
+    .col {
+    background: #f0f0f0; /* Цвет фона */
+    width: 1000px; /* Ширина блока */
+    padding: 10px; /* Поля */
+    font-size: 1.5em; /* Размер шрифта */
+    word-wrap: break-word; /* Перенос слов */
+   }
+   @keyframes spinner-border {
+    100% {
+      transform: rotate(360deg);
+    }
+  }
+
+  .submit-spinner {
+    display: inline-block;
+    width: 1rem;
+    height: 1rem;
+    vertical-align: -0.125em;
+    border: 0.2em solid currentColor;
+    border-right-color: transparent;
+    border-radius: 50%;
+    -webkit-animation: .75s linear infinite spinner-border;
+    animation: .75s linear infinite spinner-border;
+  }
+
+  .submit-spinner_hide {
+    display: none;
+  }
+</style>
+<body>
+    <form method="post" action="/submit" id="download">
+         <input type="text" name="link">
+         <button type="submit" class="custom-btn btn-1"><span class="submit-spinner submit-spinner_hide"></span> Download</button>
+    </form>
+    <div class="col">
+        <p> Ссылка для скачивания:</p>
+        <br>
+        <a id="result" href="" download></a>
+        <div class="loader">
+            <div class="loader_inner"></div>
+        </div>
+    </div>
+</body>
+<script>
+    function sendForm() {
+    const xhr = new XMLHttpRequest();
+    xhr.open('POST', document.forms.download.action);
+    xhr.responseType = 'json';
+    xhr.onload = () => {
+      document.forms.download.querySelector('[type="submit"]').disabled = false;
+      document.forms.download.querySelector('.submit-spinner').classList.add('submit-spinner_hide');
+      if (xhr.status !== 200) {
+        return;
+      }
+      const response = xhr.response;
+      result.innerHTML = xhr.response.result;
+      result.href = xhr.response.result;
+    }
+    xhr.onerror = () => {
+      document.forms.download.querySelector('[type="submit"]').disabled = false;
+      document.forms.download.querySelector('.submit-spinner').classList.add('submit-spinner_hide');
+    };
+    document.forms.download.querySelector('[type="submit"]').disabled = true;
+    document.forms.download.querySelector('.submit-spinner').classList.remove('submit-spinner_hide');
+    xhr.send(new FormData(document.forms.download));
+  }
+  // при отправке формы
+  document.forms.download.addEventListener('submit', (e) => {
+    e.preventDefault();
+    sendForm();
+  });
+</script>
+</html>