Initial commit
This commit is contained in:
commit
866fc4bb28
22 changed files with 3839 additions and 0 deletions
1
.amltconfig
Normal file
1
.amltconfig
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"project_name": "canva-render", "storage_account_name": "internblob", "container_name": "amulet", "blob_storage_account_name": "internblob", "registry_name": "projects", "local_path": "/home/v-lixinyang/canva-crawler/render", "default_output_dir": "/home/v-lixinyang/canva-crawler/render/amlt", "project_uuid": "7298582337.10185-e7711f27-f842-4c68-893c-16b1aad3197a", "version": "9.21.3"}
|
||||
2
.dockerignore
Normal file
2
.dockerignore
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
*.png
|
||||
*.cdf
|
||||
177
.gitignore
vendored
Normal file
177
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
*.json
|
||||
# Created by https://www.toptal.com/developers/gitignore/api/python
|
||||
# Edit at https://www.toptal.com/developers/gitignore?templates=python
|
||||
|
||||
### Python ###
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/#use-with-ide
|
||||
.pdm.toml
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
### Python Patch ###
|
||||
# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
|
||||
poetry.toml
|
||||
|
||||
# ruff
|
||||
.ruff_cache/
|
||||
|
||||
# LSP config files
|
||||
pyrightconfig.json
|
||||
|
||||
# End of https://www.toptal.com/developers/gitignore/api/python
|
||||
18
Dockerfile
Normal file
18
Dockerfile
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
FROM nvcr.io/nvidia/pytorch:23.10-py3
|
||||
|
||||
RUN apt update && apt install software-properties-common -y
|
||||
RUN add-apt-repository ppa:deadsnakes/ppa -y
|
||||
RUN apt update && DEBIAN_FRONTEND=noninteractive apt install curl python3.11-full python3.11-distutils python3.11-venv -y
|
||||
|
||||
ENV POETRY_HOME="/usr/local"
|
||||
RUN curl -sSL https://install.python-poetry.org | python3.11 -;
|
||||
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml /app
|
||||
RUN poetry env use /usr/bin/python3.11
|
||||
RUN poetry install
|
||||
RUN poetry run playwright install firefox && poetry run playwright install-deps
|
||||
|
||||
COPY . /app
|
||||
|
||||
CMD /bin/sh
|
||||
0
README.md
Normal file
0
README.md
Normal file
1
__init__.py
Normal file
1
__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
from render import *
|
||||
531
amlt.yaml
Normal file
531
amlt.yaml
Normal file
|
|
@ -0,0 +1,531 @@
|
|||
target:
|
||||
service: sing
|
||||
name: msrresrchvc
|
||||
|
||||
description: Render layer images
|
||||
environment:
|
||||
registry: msrresrchcr.azurecr.io
|
||||
image: v-lixinyang/canva-render:latest
|
||||
username: msrresrchcr
|
||||
|
||||
storage:
|
||||
internblob:
|
||||
storage_account_name: internblob
|
||||
container_name: v-lixinyang
|
||||
mount_dir: /mnt/v-lixinyang
|
||||
|
||||
code:
|
||||
local_dir: $CONFIG_DIR
|
||||
jobs:
|
||||
- name: job 000
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x000.json "https://www.canva.com/design/DAF1t2W8fWM/QAfCC_VopoVw9S15lT7tHw/edit?continue_in_browser=true"
|
||||
- name: job 001
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x001.json "https://www.canva.com/design/DAF1t6RtRJ8/d2dYtsTBDquX6yv0P7BDUQ/edit?continue_in_browser=true"
|
||||
- name: job 002
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x002.json "https://www.canva.com/design/DAF1t4AVupg/6lzL6W2ZJ7GSPXG305shEg/edit?continue_in_browser=true"
|
||||
- name: job 003
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x003.json "https://www.canva.com/design/DAF1twgXTPg/tDXxnayvsJOyUeToxywG1g/edit?continue_in_browser=true"
|
||||
- name: job 004
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x004.json "https://www.canva.com/design/DAF1t0Y6ozA/BbKi3f5nHieDXPQ7NW4bTw/edit?continue_in_browser=true"
|
||||
- name: job 005
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x005.json "https://www.canva.com/design/DAF1tzVk3SM/2UENqHuwg-T-zkMocAjHZg/edit?continue_in_browser=true"
|
||||
- name: job 006
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x006.json "https://www.canva.com/design/DAF1t9BRHjs/b-PEz6waxK3KSskLyxT-qw/edit?continue_in_browser=true"
|
||||
- name: job 007
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x007.json "https://www.canva.com/design/DAF1t2MYfIM/pD5R8iWgwn4r-WxmcF9Sdg/edit?continue_in_browser=true"
|
||||
- name: job 008
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x008.json "https://www.canva.com/design/DAF1twQz9k8/IZv9DuClI0UoZKz2w-FfZQ/edit?continue_in_browser=true"
|
||||
- name: job 009
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x009.json "https://www.canva.com/design/DAF1t6Vxdps/AvrSWBZU8vsyEc5Mrnw8bg/edit?continue_in_browser=true"
|
||||
- name: job 010
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x010.json "https://www.canva.com/design/DAF1txSsGLA/qzTMmacg3BXxkFmCfzO9YQ/edit?continue_in_browser=true"
|
||||
- name: job 011
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x011.json "https://www.canva.com/design/DAF1t3TGU7E/9zfUSriMfe8qgXe_b2clFg/edit?continue_in_browser=true"
|
||||
- name: job 012
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x012.json "https://www.canva.com/design/DAF1t35Q-DY/L4QflwVCwOoMkAtgr2XoKw/edit?continue_in_browser=true"
|
||||
- name: job 013
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x013.json "https://www.canva.com/design/DAF1t8ar3YI/S6tU3J_JwPLn2SUwf3EMjw/edit?continue_in_browser=true"
|
||||
- name: job 014
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x014.json "https://www.canva.com/design/DAF1t7S7KNo/koYBUsfJ9crMcuS_Wx-YEw/edit?continue_in_browser=true"
|
||||
- name: job 015
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x015.json "https://www.canva.com/design/DAF1t6K-aLc/hTCNKQkIWjrVVk6EG5sBFQ/edit?continue_in_browser=true"
|
||||
- name: job 016
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x016.json "https://www.canva.com/design/DAF1t6Ek_nM/u5FIQQ_ZwYJlOvxnEL37pw/edit?continue_in_browser=true"
|
||||
- name: job 017
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x017.json "https://www.canva.com/design/DAF1t79yBIw/qHmFFkM_8DwQotT2GLb1Vw/edit?continue_in_browser=true"
|
||||
- name: job 018
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x018.json "https://www.canva.com/design/DAF1txC5Rws/Wv_wP3Nfrt02uqe7t9SW9Q/edit?continue_in_browser=true"
|
||||
- name: job 019
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x019.json "https://www.canva.com/design/DAF1t00-GOc/CrpShiltyrfs9z2v2FpgjA/edit?continue_in_browser=true"
|
||||
- name: job 020
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x020.json "https://www.canva.com/design/DAF1t5qvyUk/bBtMWbh_TMRPfsXqELyeFw/edit?continue_in_browser=true"
|
||||
- name: job 021
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x021.json "https://www.canva.com/design/DAF1t3t5S3I/KseVhhnROHdGkB5KfuEABw/edit?continue_in_browser=true"
|
||||
- name: job 022
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x022.json "https://www.canva.com/design/DAF1twFHzPA/QyIxfzzF8pgC5fAzjJpkoQ/edit?continue_in_browser=true"
|
||||
- name: job 023
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x023.json "https://www.canva.com/design/DAF1tyF3VVs/u6zMeU9fXzH-yYyheuYNhA/edit?continue_in_browser=true"
|
||||
- name: job 024
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x024.json "https://www.canva.com/design/DAF1t70R6q0/yOXAUFV3pHKF-931QPfJ2w/edit?continue_in_browser=true"
|
||||
- name: job 025
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x025.json "https://www.canva.com/design/DAF1t6zXyBs/wZWhgFAqO0Rvu13RYFj4XA/edit?continue_in_browser=true"
|
||||
- name: job 026
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x026.json "https://www.canva.com/design/DAF1t3Q2AQY/pKqn8qjgY1SwI9KmCC2M7A/edit?continue_in_browser=true"
|
||||
- name: job 027
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x027.json "https://www.canva.com/design/DAF1t3tmwiE/iHOwhTVTNIOaTyMthxNfUw/edit?continue_in_browser=true"
|
||||
- name: job 028
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x028.json "https://www.canva.com/design/DAF1tyZmqSs/w1G-09PJ1h48_5BRrdNZgw/edit?continue_in_browser=true"
|
||||
- name: job 029
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x029.json "https://www.canva.com/design/DAF1t4VlEzs/ZZoMUzCxB_0KrewZYwZfDQ/edit?continue_in_browser=true"
|
||||
- name: job 030
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x030.json "https://www.canva.com/design/DAF1t-gXgkM/dQ9g0dbnBOHY9K235UShnQ/edit?continue_in_browser=true"
|
||||
- name: job 031
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x031.json "https://www.canva.com/design/DAF1t8I6Df4/Hbkmtm7YSmfHKEOLEir63A/edit?continue_in_browser=true"
|
||||
- name: job 032
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x032.json "https://www.canva.com/design/DAF1t0oFGWg/R11nydpqQ1A0qn55jyLGJQ/edit?continue_in_browser=true"
|
||||
- name: job 033
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x033.json "https://www.canva.com/design/DAF1t8WhEso/T3j1vTf2E6pfpYeXdzwM9g/edit?continue_in_browser=true"
|
||||
- name: job 034
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x034.json "https://www.canva.com/design/DAF1t-oo2Hc/YSqB3dONqvgeN9jnGRndzQ/edit?continue_in_browser=true"
|
||||
- name: job 035
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x035.json "https://www.canva.com/design/DAF1t4k0LSo/HlBqtYKcYKCZxcgTsLpLxQ/edit?continue_in_browser=true"
|
||||
- name: job 036
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x036.json "https://www.canva.com/design/DAF1t68vt9w/M11qS1G6HpJdPsONWSZEOA/edit?continue_in_browser=true"
|
||||
- name: job 037
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x037.json "https://www.canva.com/design/DAF1t4iTSWU/GldaZI6s6MAplvXzGLCIOQ/edit?continue_in_browser=true"
|
||||
- name: job 038
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x038.json "https://www.canva.com/design/DAF1tzsfE9o/rjSQ4hTqxgN0_r_e2uIdWg/edit?continue_in_browser=true"
|
||||
- name: job 039
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x039.json "https://www.canva.com/design/DAF1t1ZcHqs/NcWLRR0-YjKN57_F70s7Gw/edit?continue_in_browser=true"
|
||||
- name: job 040
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x040.json "https://www.canva.com/design/DAF1t9_LqkA/sSvVOS4s7iJrHV6gVtKxYg/edit?continue_in_browser=true"
|
||||
- name: job 041
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x041.json "https://www.canva.com/design/DAF1t1--24w/oTvVexQut3P6esc6ZyhtFw/edit?continue_in_browser=true"
|
||||
- name: job 042
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x042.json "https://www.canva.com/design/DAF1tzKz1XU/khG74dSStBTjyKCwEB7kkg/edit?continue_in_browser=true"
|
||||
- name: job 043
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x043.json "https://www.canva.com/design/DAF1t-89wH4/Z3s0xW_bqb6KbO-dN8w2WA/edit?continue_in_browser=true"
|
||||
- name: job 044
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x044.json "https://www.canva.com/design/DAF1txuAV-o/hT5AmLJYI7KqC0EYDzAxdQ/edit?continue_in_browser=true"
|
||||
- name: job 045
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x045.json "https://www.canva.com/design/DAF1t-VM9fg/H4inRBCd3Gbb7bxC7GqrXA/edit?continue_in_browser=true"
|
||||
- name: job 046
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x046.json "https://www.canva.com/design/DAF1t-tKYRo/lewTq3P92qvNoct1Sns-qQ/edit?continue_in_browser=true"
|
||||
- name: job 047
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x047.json "https://www.canva.com/design/DAF1twY7cso/SDWSH7B99EeDqnL1K2fdZg/edit?continue_in_browser=true"
|
||||
- name: job 048
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x048.json "https://www.canva.com/design/DAF1t8US0DM/rarmbkOF5bRHpq6HL1pPmQ/edit?continue_in_browser=true"
|
||||
- name: job 049
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x049.json "https://www.canva.com/design/DAF1txZPv_4/ud5t49ikj4LDXlVyH25BJw/edit?continue_in_browser=true"
|
||||
- name: job 050
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x050.json "https://www.canva.com/design/DAF1t3CHds4/Qu6jsNQtCokAKLhPrFpAZQ/edit?continue_in_browser=true"
|
||||
- name: job 051
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x051.json "https://www.canva.com/design/DAF1tzaPvuQ/bBejCH80HbB_H4d9kUocNg/edit?continue_in_browser=true"
|
||||
- name: job 052
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x052.json "https://www.canva.com/design/DAF1t-0b2pw/dKb0-nOnJP85CBHJ5jwYAA/edit?continue_in_browser=true"
|
||||
- name: job 053
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x053.json "https://www.canva.com/design/DAF1t5waZPw/fU8MVgGH0m3o6qSTd4jbDg/edit?continue_in_browser=true"
|
||||
- name: job 054
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x054.json "https://www.canva.com/design/DAF1tyAU2Qg/uPUw0jz6XHWnKOtrusjuIQ/edit?continue_in_browser=true"
|
||||
- name: job 055
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x055.json "https://www.canva.com/design/DAF1t9cP_-k/zZ1OXI3qC10WfBOo0lqTcQ/edit?continue_in_browser=true"
|
||||
- name: job 056
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x056.json "https://www.canva.com/design/DAF1t4uICuw/w_sqvDBVL_Vq_Zkn17gkWA/edit?continue_in_browser=true"
|
||||
- name: job 057
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x057.json "https://www.canva.com/design/DAF1t9Bqn9w/1adZ1oktHM54ArJkrSvYKg/edit?continue_in_browser=true"
|
||||
- name: job 058
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x058.json "https://www.canva.com/design/DAF1t42nA9g/2OydrAl0GdEcrhdx8iATog/edit?continue_in_browser=true"
|
||||
- name: job 059
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x059.json "https://www.canva.com/design/DAF1t_6Pofk/9bFyTCmW5avRbfZBQ6NXjg/edit?continue_in_browser=true"
|
||||
- name: job 060
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x060.json "https://www.canva.com/design/DAF1t9Ekp8U/Nl64kMLfzljnoUNuIVbd3Q/edit?continue_in_browser=true"
|
||||
- name: job 061
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x061.json "https://www.canva.com/design/DAF1t8DH6_E/Iow8JNucu4I5idZLZaM9fw/edit?continue_in_browser=true"
|
||||
- name: job 062
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x062.json "https://www.canva.com/design/DAF1t2H9JiE/S_kplr-kgxLWfQotbSH5_w/edit?continue_in_browser=true"
|
||||
- name: job 063
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x063.json "https://www.canva.com/design/DAF1t_seE_w/2r3VtE7rdDivL3leIPl6FQ/edit?continue_in_browser=true"
|
||||
- name: job 064
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x064.json "https://www.canva.com/design/DAF1t7SvTLY/NPcKvvjJQeThIW6FciBHKQ/edit?continue_in_browser=true"
|
||||
- name: job 065
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x065.json "https://www.canva.com/design/DAF1t4HcTLY/jzgFUTi0iYktyjxBNCpD0w/edit?continue_in_browser=true"
|
||||
- name: job 066
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x066.json "https://www.canva.com/design/DAF1t4BTiD4/6ebjdg99Cpq9XIioqRiqQw/edit?continue_in_browser=true"
|
||||
- name: job 067
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x067.json "https://www.canva.com/design/DAF1tznJGGI/5eR9C1ens-Dy_h5Ku6-dgg/edit?continue_in_browser=true"
|
||||
- name: job 068
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x068.json "https://www.canva.com/design/DAF1t0V5_h4/DFZqgox7fstZC9hV5eE0MA/edit?continue_in_browser=true"
|
||||
- name: job 069
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x069.json "https://www.canva.com/design/DAF1tyeUee8/XMsqfGfZNFZIg5p_K8xTug/edit?continue_in_browser=true"
|
||||
- name: job 070
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x070.json "https://www.canva.com/design/DAF1txJNI60/t-Sxe1Z6353sN7luiiTekw/edit?continue_in_browser=true"
|
||||
- name: job 071
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x071.json "https://www.canva.com/design/DAF1tyZX4Ag/k68p9NVGblWy6LCbk64Ubg/edit?continue_in_browser=true"
|
||||
- name: job 072
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x072.json "https://www.canva.com/design/DAF1t2x99oU/L90tUGZxs6-MJ_xV3_mgCA/edit?continue_in_browser=true"
|
||||
- name: job 073
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x073.json "https://www.canva.com/design/DAF1t1oDR7I/8-SgGXL9vhUlpYzZzWv45A/edit?continue_in_browser=true"
|
||||
- name: job 074
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x074.json "https://www.canva.com/design/DAF1tx5eNBM/lG3Cdyt4QAHYYoP88fumZw/edit?continue_in_browser=true"
|
||||
- name: job 075
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x075.json "https://www.canva.com/design/DAF1t1gl6QE/1MD5CLHZFI2ctSTLtvz06w/edit?continue_in_browser=true"
|
||||
- name: job 076
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x076.json "https://www.canva.com/design/DAF1t5L6S6Y/xGf3KEnJMuN4Ork34zsHbw/edit?continue_in_browser=true"
|
||||
- name: job 077
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x077.json "https://www.canva.com/design/DAF1txpwX9g/qfAoGtz29cPKflgaf8L4yQ/edit?continue_in_browser=true"
|
||||
- name: job 078
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x078.json "https://www.canva.com/design/DAF1t4YDmBs/YQE7YLf4JlwFYJ76pGlH4Q/edit?continue_in_browser=true"
|
||||
- name: job 079
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x079.json "https://www.canva.com/design/DAF1t500w1Y/oMgvj_PiThbk4qUGcs7iZA/edit?continue_in_browser=true"
|
||||
- name: job 080
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x080.json "https://www.canva.com/design/DAF1t9sVj20/Do9ze1DwEWIp5WGFs_zrgw/edit?continue_in_browser=true"
|
||||
- name: job 081
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x081.json "https://www.canva.com/design/DAF1t_QCFKg/WwXp_SBtBgeIhA6r65I1Zw/edit?continue_in_browser=true"
|
||||
- name: job 082
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x082.json "https://www.canva.com/design/DAF1t6Z2GCI/zveteCSlWFGZe-KKZpxtfw/edit?continue_in_browser=true"
|
||||
- name: job 083
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x083.json "https://www.canva.com/design/DAF1t7_DXJA/PPRCTk9EsYIQTtxiLYdnLA/edit?continue_in_browser=true"
|
||||
- name: job 084
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x084.json "https://www.canva.com/design/DAF1t3MmMFI/9vPIEMERFCtNq4g64qS4AQ/edit?continue_in_browser=true"
|
||||
- name: job 085
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x085.json "https://www.canva.com/design/DAF1t7fxfBU/X5MAofamlBVdYj_SdNwh9g/edit?continue_in_browser=true"
|
||||
- name: job 086
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x086.json "https://www.canva.com/design/DAF1twORUYI/B1oegYMDP5gg0zl1Lh5k_Q/edit?continue_in_browser=true"
|
||||
- name: job 087
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x087.json "https://www.canva.com/design/DAF1t9K2xHY/X1PcpgBuUtFeetdhApW6iw/edit?continue_in_browser=true"
|
||||
- name: job 088
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x088.json "https://www.canva.com/design/DAF1t5COMeM/roaPykIVjuZ3cSNBaA-qCQ/edit?continue_in_browser=true"
|
||||
- name: job 089
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x089.json "https://www.canva.com/design/DAF1t-pUFNk/sjulET8X4gGZ3nDH_U2BCA/edit?continue_in_browser=true"
|
||||
- name: job 090
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x090.json "https://www.canva.com/design/DAF1t84Q6T0/LuHLI1ybGWTtmMkM64QIdg/edit?continue_in_browser=true"
|
||||
- name: job 091
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x091.json "https://www.canva.com/design/DAF1tws5gBY/qf3WUqTf9EMgkTcLeaoRLw/edit?continue_in_browser=true"
|
||||
- name: job 092
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x092.json "https://www.canva.com/design/DAF1t-6UIRU/f53BCPP5llrP_8wmNTO6Vw/edit?continue_in_browser=true"
|
||||
- name: job 093
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x093.json "https://www.canva.com/design/DAF1t0e6R7s/4dB9Uzr_d5bnnB__oAN30g/edit?continue_in_browser=true"
|
||||
- name: job 094
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x094.json "https://www.canva.com/design/DAF1t2mKpWg/CFmfZGfCRBJgTNEnfv-5RQ/edit?continue_in_browser=true"
|
||||
- name: job 095
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x095.json "https://www.canva.com/design/DAF1t9HRFSg/v4vqZk_74wt9ucI2rssZ8Q/edit?continue_in_browser=true"
|
||||
- name: job 096
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x096.json "https://www.canva.com/design/DAF1t5nGqq8/dmZnBL7v9J8ezCjvv1RBIg/edit?continue_in_browser=true"
|
||||
- name: job 097
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x097.json "https://www.canva.com/design/DAF1txfTrGI/Qum6gpoLma2cmIETzfglgg/edit?continue_in_browser=true"
|
||||
- name: job 098
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x098.json "https://www.canva.com/design/DAF1t81Ozhs/dcD_8DuyyyVbF183dXn5sA/edit?continue_in_browser=true"
|
||||
- name: job 099
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x099.json "https://www.canva.com/design/DAF1tyzsGXc/aq5gz7hfxQY9VDupY2039g/edit?continue_in_browser=true"
|
||||
- name: job 100
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x100.json "https://www.canva.com/design/DAF1t_50qvo/HAWouQcX9Abc4BrSSq8PYA/edit?continue_in_browser=true"
|
||||
- name: job 101
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x101.json "https://www.canva.com/design/DAF1t2W8fWM/QAfCC_VopoVw9S15lT7tHw/edit?continue_in_browser=true"
|
||||
- name: job 102
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x102.json "https://www.canva.com/design/DAF1t6RtRJ8/d2dYtsTBDquX6yv0P7BDUQ/edit?continue_in_browser=true"
|
||||
- name: job 103
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x103.json "https://www.canva.com/design/DAF1t4AVupg/6lzL6W2ZJ7GSPXG305shEg/edit?continue_in_browser=true"
|
||||
- name: job 104
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x104.json "https://www.canva.com/design/DAF1twgXTPg/tDXxnayvsJOyUeToxywG1g/edit?continue_in_browser=true"
|
||||
- name: job 105
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x105.json "https://www.canva.com/design/DAF1t0Y6ozA/BbKi3f5nHieDXPQ7NW4bTw/edit?continue_in_browser=true"
|
||||
- name: job 106
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x106.json "https://www.canva.com/design/DAF1tzVk3SM/2UENqHuwg-T-zkMocAjHZg/edit?continue_in_browser=true"
|
||||
- name: job 107
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x107.json "https://www.canva.com/design/DAF1t9BRHjs/b-PEz6waxK3KSskLyxT-qw/edit?continue_in_browser=true"
|
||||
- name: job 108
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x108.json "https://www.canva.com/design/DAF1t2MYfIM/pD5R8iWgwn4r-WxmcF9Sdg/edit?continue_in_browser=true"
|
||||
- name: job 109
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x109.json "https://www.canva.com/design/DAF1twQz9k8/IZv9DuClI0UoZKz2w-FfZQ/edit?continue_in_browser=true"
|
||||
- name: job 110
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x110.json "https://www.canva.com/design/DAF1t6Vxdps/AvrSWBZU8vsyEc5Mrnw8bg/edit?continue_in_browser=true"
|
||||
- name: job 111
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x111.json "https://www.canva.com/design/DAF1txSsGLA/qzTMmacg3BXxkFmCfzO9YQ/edit?continue_in_browser=true"
|
||||
- name: job 112
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x112.json "https://www.canva.com/design/DAF1t3TGU7E/9zfUSriMfe8qgXe_b2clFg/edit?continue_in_browser=true"
|
||||
- name: job 113
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x113.json "https://www.canva.com/design/DAF1t35Q-DY/L4QflwVCwOoMkAtgr2XoKw/edit?continue_in_browser=true"
|
||||
- name: job 114
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x114.json "https://www.canva.com/design/DAF1t8ar3YI/S6tU3J_JwPLn2SUwf3EMjw/edit?continue_in_browser=true"
|
||||
- name: job 115
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x115.json "https://www.canva.com/design/DAF1t7S7KNo/koYBUsfJ9crMcuS_Wx-YEw/edit?continue_in_browser=true"
|
||||
- name: job 116
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x116.json "https://www.canva.com/design/DAF1t6K-aLc/hTCNKQkIWjrVVk6EG5sBFQ/edit?continue_in_browser=true"
|
||||
- name: job 117
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x117.json "https://www.canva.com/design/DAF1t6Ek_nM/u5FIQQ_ZwYJlOvxnEL37pw/edit?continue_in_browser=true"
|
||||
- name: job 118
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x118.json "https://www.canva.com/design/DAF1t79yBIw/qHmFFkM_8DwQotT2GLb1Vw/edit?continue_in_browser=true"
|
||||
- name: job 119
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x119.json "https://www.canva.com/design/DAF1txC5Rws/Wv_wP3Nfrt02uqe7t9SW9Q/edit?continue_in_browser=true"
|
||||
- name: job 120
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x120.json "https://www.canva.com/design/DAF1t00-GOc/CrpShiltyrfs9z2v2FpgjA/edit?continue_in_browser=true"
|
||||
- name: job 121
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x121.json "https://www.canva.com/design/DAF1t5qvyUk/bBtMWbh_TMRPfsXqELyeFw/edit?continue_in_browser=true"
|
||||
- name: job 122
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x122.json "https://www.canva.com/design/DAF1t3t5S3I/KseVhhnROHdGkB5KfuEABw/edit?continue_in_browser=true"
|
||||
- name: job 123
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x123.json "https://www.canva.com/design/DAF1twFHzPA/QyIxfzzF8pgC5fAzjJpkoQ/edit?continue_in_browser=true"
|
||||
- name: job 124
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x124.json "https://www.canva.com/design/DAF1tyF3VVs/u6zMeU9fXzH-yYyheuYNhA/edit?continue_in_browser=true"
|
||||
- name: job 125
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x125.json "https://www.canva.com/design/DAF1t70R6q0/yOXAUFV3pHKF-931QPfJ2w/edit?continue_in_browser=true"
|
||||
- name: job 126
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x126.json "https://www.canva.com/design/DAF1t6zXyBs/wZWhgFAqO0Rvu13RYFj4XA/edit?continue_in_browser=true"
|
||||
- name: job 127
|
||||
sku: C15
|
||||
command:
|
||||
- sh run.sh /mnt/v-lixinyang/cdf-11.30/x127.json "https://www.canva.com/design/DAF1t3Q2AQY/pKqn8qjgY1SwI9KmCC2M7A/edit?continue_in_browser=true"
|
||||
137
cdf_parser.py
Normal file
137
cdf_parser.py
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, asdict
|
||||
|
||||
import logging
|
||||
from typing import Tuple
|
||||
from copy import deepcopy
|
||||
|
||||
from cv2 import merge
|
||||
|
||||
with open("../canva.fonts.json", "r") as f:
|
||||
fonts = json.load(f)
|
||||
fonts = {f"{font['_id']},{font['index']}": font for font in fonts}
|
||||
|
||||
def catch_keyerror(default):
|
||||
def _catch_keyerror(func):
|
||||
def wrapper(*args, **kwargs):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except KeyError as e:
|
||||
logging.warning(f"Key {e} not found at {func.__name__}({args[0].id})")
|
||||
return default
|
||||
return wrapper
|
||||
return _catch_keyerror
|
||||
|
||||
@dataclass
|
||||
class TextElement():
|
||||
text: str
|
||||
position: Tuple[float, float, float, float]
|
||||
label: str
|
||||
style: dict
|
||||
|
||||
@dataclass
|
||||
class TextStyle():
|
||||
transform: dict
|
||||
|
||||
|
||||
class CdfParser(object):
|
||||
def __init__(self, cdf, id):
|
||||
# self.data = json.loads(cdf)
|
||||
self.data = cdf
|
||||
self.id = id
|
||||
|
||||
# FIXME: Not all cdf have title
|
||||
@catch_keyerror(None)
|
||||
def get_title(self):
|
||||
return self.data['content']['D']
|
||||
|
||||
@catch_keyerror([])
|
||||
def get_elements(self):
|
||||
if len(self.data['content']['A']) > 1:
|
||||
logging.warning(f"More than 1 layout at {self.id}")
|
||||
elements = self.data['content']['A'][0]['E']
|
||||
if elements is None:
|
||||
return []
|
||||
else:
|
||||
return elements
|
||||
|
||||
@staticmethod
|
||||
def get_text(element):
|
||||
position = (element['A'], element['B'], element['C'], element['D'])
|
||||
text = element['a']['A'][0]['A']
|
||||
label = element.get('N')
|
||||
merged_style = {}
|
||||
styles = element['a']['B']
|
||||
if styles is None:
|
||||
logging.warning("Empty font styles")
|
||||
return None
|
||||
for style in element['a']['B']:
|
||||
if style['A?'] != 'A':
|
||||
continue
|
||||
merged_style.update({
|
||||
attribute_name: list(attr_value_dict.values())[0]
|
||||
for attribute_name, attr_value_dict in style['A'].items()
|
||||
})
|
||||
try:
|
||||
merged_style['font-info'] = fonts[merged_style['font-family']]
|
||||
except:
|
||||
pass
|
||||
return TextElement(text, position,label, merged_style)
|
||||
|
||||
|
||||
def get_texts(self):
|
||||
res = []
|
||||
|
||||
for index, element in enumerate(self.get_elements()[::-1]):
|
||||
# Enumerate backwards to align with pop() in remove_elements_iter()
|
||||
if element['A?'] == 'K':
|
||||
if len(element['a']['A']) > 1:
|
||||
logging.warning(f"More than 1 text element at {self.id}")
|
||||
breakpoint()
|
||||
text_groups = self.get_text(element)
|
||||
if text_groups is None:
|
||||
continue
|
||||
text_groups = asdict(text_groups)
|
||||
text_groups['index'] = index
|
||||
res.append(text_groups)
|
||||
elif element['A?'] == 'H':
|
||||
subres = []
|
||||
for child_element in element['c']:
|
||||
if child_element['A?'] == "K":
|
||||
text_groups = self.get_text(child_element)
|
||||
if text_groups is None:
|
||||
continue
|
||||
text_groups = asdict(text_groups)
|
||||
text_groups['index'] = index
|
||||
subres.append(text_groups)
|
||||
if subres:
|
||||
res.append(subres)
|
||||
return res
|
||||
|
||||
def get_data(self):
|
||||
return self.data
|
||||
|
||||
def remote_texts(self):
|
||||
for element in self.get_elements():
|
||||
if element['A?'] == "K":
|
||||
element['a']['A'][0]['A'] = "" # clear texts
|
||||
elif element['A?'] == "H": # Grouped text
|
||||
for child in element['c']:
|
||||
if child['A?'] == "K":
|
||||
child['a']['A'][0]['A'] = ""
|
||||
else:
|
||||
continue
|
||||
|
||||
# remove elements one by one and return a generator of self
|
||||
def remove_elements_iter(self):
|
||||
elements = self.get_elements()
|
||||
if len(elements) == 0:
|
||||
return None
|
||||
while elements:
|
||||
elements.pop()
|
||||
yield deepcopy(self.data)
|
||||
|
||||
if __name__ == "__main__":
|
||||
t = TextElement("abc", (1, 1, 1, 1), "label", {"s": "style"})
|
||||
breakpoint()
|
||||
78
deploy.sh
Normal file
78
deploy.sh
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
ssh_start_port=12001
|
||||
deply_start_port=8001
|
||||
bastion_list="10
|
||||
12
|
||||
13
|
||||
15
|
||||
11
|
||||
17
|
||||
18
|
||||
25"
|
||||
|
||||
project_urls="https://www.canva.com/design/DAFxmaf2y_Y/VMZhoqHPmqCV4zoDmkhzaw/edit
|
||||
https://www.canva.com/design/DAFxmcF74iw/NRJcb0bvg0kN25HSUvWTPw/edit
|
||||
https://www.canva.com/design/DAFxmVPAylk/JQCYHj2fE7qwMbvFstNptw/edit
|
||||
https://www.canva.com/design/DAFxmcQATfQ/ZnbWUc_6so4yH1LzK9bwcA/edit
|
||||
https://www.canva.com/design/DAFxmeOxIMo/Ucfx9UDFhg0RYx9w9HKiTw/edit
|
||||
https://www.canva.com/design/DAFxmcvwu_E/lw-7caSNYuS6UlGr_rc8yQ/edit
|
||||
https://www.canva.com/design/DAFxmfTKoe4/seL09fsTLv_n8Jm-HsfvYA/edit
|
||||
https://www.canva.com/design/DAFxmUgjAkk/mIGcfJe8PWHFUBgzQSQ-mg/edit"
|
||||
|
||||
runtime=nerdctl
|
||||
|
||||
do_ssh () {
|
||||
# $1: port
|
||||
# $2: command
|
||||
ssh -o StrictHostKeyChecking=no -p $1 REDMOND.v-lixinyang@localhost -- "$2"
|
||||
}
|
||||
|
||||
create_tunnel () {
|
||||
ssh_port=$ssh_start_port
|
||||
|
||||
for bastion in $bastion_list
|
||||
do
|
||||
sudo fuser -k $ssh_port/tcp
|
||||
sudo az network bastion tunnel --subscription 7ccdb8ae-4daf-4f0f-8019-e80665eb00d2 --name CPU-Sandbox-VNET-bastion --resource-group CPU-SANDBOX --target-resource-id /subscriptions/7ccdb8ae-4daf-4f0f-8019-e80665eb00d2/resourceGroups/CPU-SANDBOX/providers/Microsoft.Compute/virtualMachines/GCRAZCDL00$bastion --resource-port 22 --port $ssh_port &
|
||||
ssh_port=$((ssh_port + 1));
|
||||
done
|
||||
sleep 10
|
||||
}
|
||||
|
||||
do_create_docker () {
|
||||
create_tunnel
|
||||
|
||||
ssh_port=$ssh_start_port
|
||||
for url in $project_urls
|
||||
do
|
||||
do_ssh $ssh_port "sudo $runtime rm --force canva-render; sudo $runtime pull msroctocr.azurecr.io/v-lixinyang/canva-render; sudo $runtime run --network=host -d -e CANVA_EDIT_URL=\"$url\" --name=canva-render msroctocr.azurecr.io/v-lixinyang/canva-render;"
|
||||
ssh_port=$((ssh_port + 1))
|
||||
done
|
||||
}
|
||||
|
||||
do_pull_log () {
|
||||
create_tunnel
|
||||
|
||||
ssh_port=$ssh_start_port
|
||||
for url in $project_urls
|
||||
do
|
||||
do_ssh $ssh_port "sudo $runtime logs --tail 40 canva-render"
|
||||
ssh_port=$((ssh_port + 1))
|
||||
done
|
||||
}
|
||||
|
||||
do_install_nerdctl () {
|
||||
create_tunnel
|
||||
|
||||
ssh_port=$ssh_start_port
|
||||
for url in $project_urls
|
||||
do
|
||||
do_ssh $ssh_port "wget https://github.com/containerd/nerdctl/releases/download/v1.6.2/nerdctl-1.6.2-linux-amd64.tar.gz; tar xvf nerdctl-1.6.2-linux-amd64.tar.gz; sudo mv nerdctl /usr/bin"
|
||||
ssh_port=$((ssh_port + 1))
|
||||
done
|
||||
}
|
||||
|
||||
set -x
|
||||
|
||||
# do_install_nerdctl
|
||||
# do_create_docker
|
||||
do_pull_log
|
||||
1998
poetry.lock
generated
Normal file
1998
poetry.lock
generated
Normal file
File diff suppressed because it is too large
Load diff
24
pyproject.toml
Normal file
24
pyproject.toml
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
[tool.poetry]
|
||||
name = "render"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = ["Your Name <you@example.com>"]
|
||||
readme = "README.md"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.11"
|
||||
playwright = "^1.38.0"
|
||||
pymongo = "^4.5.0"
|
||||
fastapi = {extras = ["full"], version = "^0.103.2"}
|
||||
uvicorn = "^0.23.2"
|
||||
jinja2 = "^3.1.0"
|
||||
opencv-python = "^4.8.1.78"
|
||||
azure-storage-blob = "^12.18.3"
|
||||
motor = "^3.3.1"
|
||||
aiohttp = "^3.8.6"
|
||||
scrapy = "^2.11.0"
|
||||
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
194
render.py
Normal file
194
render.py
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
# from playwright.sync_api import sync_playwright
|
||||
from playwright.async_api import async_playwright, Playwright, Browser, BrowserContext, Page, Route
|
||||
from collections import Counter
|
||||
from functools import partial
|
||||
from typing import Optional
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import logging
|
||||
import uuid
|
||||
import asyncio
|
||||
import copy
|
||||
|
||||
class RoutePage(object):
|
||||
|
||||
def __init__(self, page: Page, route, func):
|
||||
self.page = page
|
||||
|
||||
def __enter__(self):
|
||||
self.page.route(self.route, self.func)
|
||||
|
||||
def __exit__(self):
|
||||
self.page.unroute(self.route)
|
||||
|
||||
class Renderer:
|
||||
# TODO: URL decide the size of canvas
|
||||
EDIT_URL = os.environ.get("CANVA_EDIT_URL", "")
|
||||
BLOCKED_URL = [
|
||||
r"www.canva.com/_ajax/ae/createBatch.*",
|
||||
r"telemetry.canva.com/v1/traces.*",
|
||||
r"www.canva.com/_ajax/search/media/usage.*",
|
||||
r"www.canva.com/_ajax/reaction/reactions/summaries.*"
|
||||
r"www.canva.com/_ajax/search/templates.*",
|
||||
r"www.canva.com/_ajax/search/related-items.*",
|
||||
r"www.canva.com/_ajax/profile/v2/brands/summary.*",
|
||||
r"sentry.io/api/1766513/envelope.*",
|
||||
r"www.canva.com/cdn-cgi/rum.*",
|
||||
r"www.canva.com/_ajax/usage/struct.*",
|
||||
r"www.canva.com/_ajax/alerts.*",
|
||||
]
|
||||
def __init__(self, storage_state: str = "me@xinyang.life.json", debug: bool = False, concurrent_workers: int = 1):
|
||||
self.storage_state = storage_state
|
||||
self.browser: Optional[Browser] = None
|
||||
self.context: Optional[BrowserContext] = None
|
||||
self.queue: asyncio.Queue[tuple[bytes, asyncio.Queue]] = asyncio.Queue(maxsize=1000)
|
||||
self.debug = debug
|
||||
self.logger = logging.getLogger(__name__)
|
||||
self.concurrent_workers = concurrent_workers
|
||||
self.worker_pages = dict()
|
||||
|
||||
async def render_page(self, cdf: bytes):
|
||||
result = asyncio.Queue(maxsize=1)
|
||||
await self.queue.put((cdf, result))
|
||||
page: Optional[bytes] = copy.deepcopy(await result.get())
|
||||
del result
|
||||
return page
|
||||
|
||||
async def worker_screenshot(self, worker_id: int) -> bytes:
|
||||
return await self.worker_pages[worker_id].screenshot()
|
||||
|
||||
async def worker_content(self, worker_id: int) -> bytes:
|
||||
return await self.worker_pages[worker_id].content()
|
||||
|
||||
async def run(self):
|
||||
async with async_playwright() as p:
|
||||
self.browser = await p.firefox.launch()
|
||||
self.context = await self.browser.new_context(storage_state=self.storage_state, viewport={"width": 2000, "height": 2500})
|
||||
tasks = set()
|
||||
for worker_id in range(self.concurrent_workers):
|
||||
tasks.add(self._workers(worker_id, delay=worker_id * 15))
|
||||
try:
|
||||
await asyncio.gather(*tasks, return_exceptions=False)
|
||||
except asyncio.CancelledError:
|
||||
self.logger.error("a worker cancelled")
|
||||
|
||||
async def _workers(self, worker_id: int, delay: float = 0):
|
||||
await asyncio.sleep(delay)
|
||||
page = await self._new_page()
|
||||
self.worker_pages[worker_id] = page
|
||||
self.logger.info(f"Created page in worker {worker_id}")
|
||||
logger = self.logger.getChild(f"worker-{worker_id}")
|
||||
remain_time = 100000 * 60 # Won't reach memory limit on Singularity, so disable this
|
||||
start_time = time.time()
|
||||
while True:
|
||||
cdf, result = await self.queue.get()
|
||||
logger.info(f"Got cdf from queue")
|
||||
# Restart worker every 30 minutes to avoid memory leak
|
||||
if time.time() - start_time > remain_time:
|
||||
await page.close()
|
||||
page = await self._new_page()
|
||||
self.worker_pages[worker_id] = page
|
||||
logger.info(f"Restart page in worker {worker_id}")
|
||||
start_time = time.time()
|
||||
try:
|
||||
await result.put(await self._render(page, cdf, logger=logger))
|
||||
logger.info(f"Put result to queue")
|
||||
except Exception as e:
|
||||
logger.error(f"Error: {e}")
|
||||
await result.put(None)
|
||||
self.queue.task_done()
|
||||
|
||||
async def _new_page(self):
|
||||
assert self.context is not None
|
||||
page = await self.context.new_page()
|
||||
|
||||
# Disable websocket here!
|
||||
await page.add_init_script("WebSocket = () => {};")
|
||||
|
||||
await page.goto(self.EDIT_URL)
|
||||
|
||||
# Wait for search panel to be loaded
|
||||
await page.wait_for_timeout(10000)
|
||||
|
||||
if self.debug:
|
||||
await page.screenshot(path="start_page.png")
|
||||
|
||||
await page.get_by_role("tab", name="Starred").click()
|
||||
# make sure you got something in starred tab
|
||||
self.template_selector = page.get_by_label("Black Million Waves Logo").first
|
||||
await self.template_selector.click()
|
||||
await page.wait_for_timeout(5000)
|
||||
|
||||
await page.locator("css=.HTh_Cg").first.click(force=True, position={"x": 0, "y": 0})
|
||||
await page.wait_for_timeout(50)
|
||||
self.init_render = await page.locator("css=._2y_DBA").first.screenshot()
|
||||
|
||||
await self.block(page, self.BLOCKED_URL)
|
||||
# await page.route(re.compile(r"https://www.canva.com/_online/.*"), lambda x: x.fulfill(status=200, body=b""))
|
||||
# await self.context.storage_state(path="me@xinyang.life.json")
|
||||
await page.screenshot(path="final_start_page.png")
|
||||
return page
|
||||
|
||||
async def _render(self, page: Page, cdf: bytes, logger: Optional[logging.Logger] = None) -> Optional[bytes]:
|
||||
transaction_id = uuid.uuid4()
|
||||
if logger is None:
|
||||
logger = self.logger
|
||||
logger.info(f"start rendering page")
|
||||
with RoutePage(page, re.compile(r"https://template.canva.com/.*\.cdf"), partial(self.handle, cdf)) as r:
|
||||
try:
|
||||
await page.get_by_label("Black Million Waves Logo", exact=True).click()
|
||||
except Exception as e:
|
||||
logger.error(f"Exception({transaction_id}): {e} (trying to click continue editing)")
|
||||
await page.screenshot(path=f"{transaction_id}-error-open-template.png")
|
||||
try:
|
||||
await page.get_by_text("continue editing").click()
|
||||
await page.get_by_label("Black Million Waves Logo", exact=True).click()
|
||||
except Exception as e:
|
||||
await page.get_by_text("continue editing").highlight()
|
||||
logger.error(f"Exception({transaction_id}): no `continue editing` button")
|
||||
await page.screenshot(path=f"{transaction_id}-error-continue-editing.png")
|
||||
return None
|
||||
|
||||
logger.info(f"template item opened")
|
||||
|
||||
try:
|
||||
await page.get_by_role("button", name="Replace current page").click(timeout=5000)
|
||||
logger.info("Replace current page popup, clicked")
|
||||
await page.get_by_label("Black Million Waves Logo", exact=True).click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# try:
|
||||
# await page.locator("css=.yzhVgQ").nth(1).wait_for(state="attached")
|
||||
# except Exception as e:
|
||||
# logger.error(f"Exception({transaction_id}): {e}")
|
||||
# await page.screenshot(path=f"{transaction_id}-error-yzhVgQ.png")
|
||||
await page.wait_for_timeout(8000)
|
||||
|
||||
await page.locator("css=.HTh_Cg").first.click(force=True, position={"x": 0, "y": 0})
|
||||
await page.wait_for_timeout(50)
|
||||
result = await page.locator("css=._2y_DBA").first.screenshot()
|
||||
logger.info(f"screenshot taken")
|
||||
|
||||
# Assert the new page is different from init_render
|
||||
if result == self.init_render:
|
||||
logger.error(f"Exception({transaction_id}): result is the same as init_render")
|
||||
return None
|
||||
|
||||
return result
|
||||
|
||||
async def block(self, page: Page, block_list=[]):
|
||||
async def log_abort(route: Route):
|
||||
# self.logger.info(f"BLOCKED {route.request.url}")
|
||||
await route.abort()
|
||||
for url in block_list:
|
||||
await page.route(re.compile(url), log_abort)
|
||||
|
||||
async def unblock(self, page: Page, block_list=[]):
|
||||
for url in block_list:
|
||||
await page.route(re.compile(url), lambda x: x.continue_())
|
||||
|
||||
async def handle(self, cdf: bytes, route: Route):
|
||||
self.logger.info(f"HIJACKED {route.request.url}")
|
||||
await route.fulfill(body=cdf)
|
||||
228
render_controller.py
Normal file
228
render_controller.py
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import re
|
||||
from copy import deepcopy
|
||||
from collections import defaultdict
|
||||
from dataclasses import asdict, dataclass
|
||||
from io import BytesIO
|
||||
from typing import Optional
|
||||
|
||||
import cv2
|
||||
import scrapy
|
||||
import numpy as np
|
||||
from azure.storage.blob import BlobServiceClient
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.exceptions import DropItem
|
||||
|
||||
from cdf_parser import CdfParser
|
||||
|
||||
|
||||
def get_mask(img1: np.ndarray, img2: np.ndarray):
|
||||
"""Assume img1 and img2 are exactly the same, except text areas
|
||||
"""
|
||||
try:
|
||||
diff = cv2.absdiff(img1, img2)
|
||||
except:
|
||||
raise ValueError("img1 and img2 are not the same size")
|
||||
mask = cv2.cvtColor(diff, cv2.COLOR_RGBA2GRAY)
|
||||
thresh, binmask = cv2.threshold(mask, 10, 255, cv2.THRESH_BINARY)
|
||||
return thresh, binmask
|
||||
|
||||
@dataclass
|
||||
class RenderedImage:
|
||||
_id: str
|
||||
image: bytes
|
||||
cdf_with_metadata: dict
|
||||
total_elements: int
|
||||
element_idx: Optional[int]
|
||||
|
||||
@dataclass
|
||||
class RenderedImageWithMask(RenderedImage):
|
||||
mask: bytes
|
||||
|
||||
class QuietLogFormatter(scrapy.logformatter.LogFormatter):
|
||||
def dropped(self, item, exception, response, spider):
|
||||
return {
|
||||
'level': logging.INFO, # lowering the level from logging.WARNING
|
||||
'msg': "Dropped: %(exception)s",
|
||||
'args': {
|
||||
'exception': exception,
|
||||
}
|
||||
}
|
||||
|
||||
class GroupAndFilterPipeline:
|
||||
unpaired: dict[str, dict[int, RenderedImage]] = defaultdict(dict)
|
||||
|
||||
@staticmethod
|
||||
def mask_discriminator(img1, img2, mask, thresh=0.5):
|
||||
return True
|
||||
non_zero_pixels = cv2.countNonZero(mask)
|
||||
total_pixels = mask.shape[0] * mask.shape[1]
|
||||
if non_zero_pixels > total_pixels * thresh:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
def process_item(self, item: RenderedImage, spider):
|
||||
if item.element_idx is None:
|
||||
# whole image as -1 to keep order
|
||||
self.unpaired[item._id][-1] = item
|
||||
else:
|
||||
self.unpaired[item._id][item.element_idx] = item
|
||||
|
||||
rendered_items = self.unpaired[item._id]
|
||||
|
||||
if len(rendered_items) - 1 == item.total_elements:
|
||||
# No more layers
|
||||
for i in range(-1, item.total_elements):
|
||||
if i not in rendered_items:
|
||||
# Missing layer, put back to queue
|
||||
spider.cdf_queue.put(item.cdf_with_metadata)
|
||||
raise DropItem(f"Missing layer {i} for {item._id}, layer index must cover [-1, {item.total_elements})")
|
||||
diff_masks = dict()
|
||||
for i in range(item.total_elements):
|
||||
img1 = cv2.imdecode(np.asarray(bytearray(rendered_items[i-1].image), dtype=np.uint8), cv2.IMREAD_ANYCOLOR)
|
||||
img2 = cv2.imdecode(np.asarray(bytearray(rendered_items[i].image), dtype=np.uint8), cv2.IMREAD_ANYCOLOR)
|
||||
try:
|
||||
_, mask = get_mask(img1, img2)
|
||||
except ValueError:
|
||||
# Error in get_mask, put back to queue
|
||||
spider.cdf_queue.put(item.cdf_with_metadata)
|
||||
raise DropItem(f"Error in get_mask for {item._id} between {i} and {i-1}")
|
||||
if self.mask_discriminator(img1, img2, mask):
|
||||
diff_masks[i] = mask
|
||||
else:
|
||||
# Render error
|
||||
spider.cdf_queue.put(item.cdf_with_metadata)
|
||||
raise DropItem(f"Discriminator failed for {item._id} between {i} and {i-1}")
|
||||
return [ rendered_items[-1] ]+ [
|
||||
RenderedImageWithMask(
|
||||
mask=cv2.imencode('.png', mask)[1].tobytes(),
|
||||
**asdict(rendered_items[i])
|
||||
)
|
||||
for i, mask in diff_masks.items()
|
||||
]
|
||||
else:
|
||||
idx = -1 if item.element_idx is None else item.element_idx
|
||||
self.unpaired[item._id][idx] = item
|
||||
return None
|
||||
|
||||
class AzureUploadPipeline:
|
||||
|
||||
AZUREBLOB_SAS_URL = "https://internblob.blob.core.windows.net/v-lixinyang?sp=racwdli&st=2023-10-08T06:40:24Z&se=2024-01-04T14:40:24Z&spr=https&sv=2022-11-02&sr=c&sig=77O5xpepaRehrwUJ0FSnYQDbTBJzxg629GpStnWrFg4%3D"
|
||||
CONTAINER = "canva-render-11.30"
|
||||
|
||||
def process_item(self, items: Optional[list[RenderedImageWithMask]], spider):
|
||||
if items is None:
|
||||
return None
|
||||
|
||||
for item in items:
|
||||
if item.element_idx is None:
|
||||
filename = f"{item._id}-full.png"
|
||||
self.client.upload_blob(name=filename, data=BytesIO(item.image), overwrite=True)
|
||||
else:
|
||||
filename = f"{item._id}-({item.element_idx}).png"
|
||||
filename_mask = f"{item._id}-({item.element_idx})-mask.png"
|
||||
self.client.upload_blob(name=filename, data=BytesIO(item.image), overwrite=True)
|
||||
self.client.upload_blob(name=filename_mask, data=BytesIO(item.mask), overwrite=True)
|
||||
spider.logger.info(f"Uploaded {items[0]._id} to Azure Blob Storage")
|
||||
return items
|
||||
|
||||
def open_spider(self, spider):
|
||||
# Acquire the logger for azure sdk library
|
||||
logger = logging.getLogger('azure.mgmt.resource')
|
||||
# Set the desired logging level
|
||||
logger.setLevel(logging.WARNING)
|
||||
|
||||
blob_client = BlobServiceClient(account_url=self.AZUREBLOB_SAS_URL, logger=logger)
|
||||
self.client = blob_client.get_container_client(self.CONTAINER)
|
||||
|
||||
|
||||
class ImagesCrawler(scrapy.Spider):
|
||||
name = "render-controller"
|
||||
custom_settings = {
|
||||
"LOG_LEVEL": "INFO",
|
||||
"LOG_FORMATTER": "__main__.QuietLogFormatter",
|
||||
"ITEM_PIPELINES": {
|
||||
"__main__.GroupAndFilterPipeline": 300,
|
||||
"__main__.AzureUploadPipeline": 400,
|
||||
},
|
||||
# "DOWNLOAD_DELAY": 1,
|
||||
"CONCURRENT_REQUESTS": 8,
|
||||
}
|
||||
cdf_file = "cdfs.json"
|
||||
cdf_queue = queue.Queue()
|
||||
|
||||
def start_requests(self):
|
||||
self.logger.info(f"Reading cdf file {self.cdf_file}")
|
||||
with open(self.cdf_file, "r") as f:
|
||||
for line in f:
|
||||
cdf_with_metadata = json.loads(line)
|
||||
self.cdf_queue.put(cdf_with_metadata)
|
||||
|
||||
while True:
|
||||
try:
|
||||
cdf_with_metadata = self.cdf_queue.get(timeout=10)
|
||||
except TimeoutError:
|
||||
break
|
||||
assert (match := re.match(r"^https://template.canva.com/[-_\w]{1,11}/(\d+)/(\d+)/(.*)\.cdf$", cdf_with_metadata["url"]))
|
||||
num1, num2, tid = match.groups()
|
||||
|
||||
cdf_id = f"{cdf_with_metadata['template_id']}-{num1}-{num2}-{tid}"
|
||||
|
||||
cdf_parser = CdfParser(cdf_with_metadata["content"], cdf_with_metadata["template_id"])
|
||||
total_elements = len(cdf_parser.get_elements())
|
||||
|
||||
# Render whole image with all elements
|
||||
yield scrapy.Request(
|
||||
f"http://localhost:8000/render",
|
||||
method='POST',
|
||||
body=json.dumps(cdf_with_metadata["content"]),
|
||||
callback=self.parse,
|
||||
cb_kwargs=dict(
|
||||
id=cdf_id,
|
||||
cdf_with_metadata=cdf_with_metadata,
|
||||
total_elements=total_elements,
|
||||
element_idx=None
|
||||
),
|
||||
headers={'Content-Type':'application/json'}
|
||||
)
|
||||
|
||||
layered_cdf_with_metadata = deepcopy(cdf_with_metadata)
|
||||
layered_cdf_parser = CdfParser(layered_cdf_with_metadata["content"], layered_cdf_with_metadata["template_id"])
|
||||
|
||||
for element_idx, layer_cdf in enumerate(layered_cdf_parser.remove_elements_iter()):
|
||||
yield scrapy.Request(
|
||||
f"http://localhost:8000/render",
|
||||
method='POST',
|
||||
body=json.dumps(layer_cdf),
|
||||
callback=self.parse,
|
||||
cb_kwargs=dict(
|
||||
id=cdf_id,
|
||||
cdf_with_metadata=cdf_with_metadata,
|
||||
total_elements=total_elements,
|
||||
element_idx=element_idx,
|
||||
),
|
||||
headers={'Content-Type':'application/json'}
|
||||
)
|
||||
|
||||
def parse(self, response, id, cdf_with_metadata, total_elements, element_idx):
|
||||
yield RenderedImage(
|
||||
_id=id,
|
||||
cdf_with_metadata=cdf_with_metadata,
|
||||
image=response.body,
|
||||
total_elements=total_elements,
|
||||
element_idx=element_idx,
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-f", "--cdf-file")
|
||||
args = parser.parse_args()
|
||||
|
||||
process = CrawlerProcess()
|
||||
process.crawl(ImagesCrawler, cdf_file=args.cdf_file)
|
||||
process.start()
|
||||
10
run.sh
Normal file
10
run.sh
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
#!/bin/sh
|
||||
|
||||
pwd
|
||||
ls
|
||||
poetry env use /usr/bin/python3.11
|
||||
poetry install
|
||||
poetry run playwright install firefox && poetry run playwright install-deps
|
||||
CANVA_EDIT_URL=$2 poetry run python serve.py &
|
||||
sleep 180
|
||||
# poetry run python render_controller.py -f $1
|
||||
63
scripts/gen_meta.py
Normal file
63
scripts/gen_meta.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
import json
|
||||
import cv2
|
||||
import pprint
|
||||
import os.path as osp
|
||||
from cdf_parser import CdfParser
|
||||
|
||||
print("""
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
.row {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
# flex-wrap: wrap;
|
||||
justify-content: start;
|
||||
align-items: center;
|
||||
overflow-x: auto;
|
||||
margin-bottom: 16px;
|
||||
align-items: start;
|
||||
}
|
||||
.column {
|
||||
flex: 25%;
|
||||
padding: 0 1px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
""")
|
||||
|
||||
|
||||
path = "https://internblob.blob.core.windows.net/v-lixinyang/canva-render-11.30/{}?sp=racwdli&st=2023-09-17T15:37:58Z&se=2023-12-31T23:37:58Z&spr=https&sv=2022-11-02&sr=c&sig=u%2FPbZ4fNttAPeLj0NEEpX0eIgFcjhot%2Bmy3iGd%2BCmxk%3D"
|
||||
with open("cdfs.json", "r") as f:
|
||||
for line in f:
|
||||
cdf = json.loads(line)
|
||||
id = cdf['rendered_folder']
|
||||
cdf_parser = CdfParser(cdf['content'], id)
|
||||
elements = cdf_parser.get_elements()
|
||||
print('<div class="row">')
|
||||
elements = [e for e in elements[::-1]]
|
||||
for index, element in enumerate(elements):
|
||||
name = 'full' if index == 0 else f"({index - 1})"
|
||||
element_text = json.dumps(element, indent=2).replace("\n", "<br/>").replace(" ", " "*2)
|
||||
print(f"""
|
||||
<div class="column">
|
||||
<img src="{path.format(id + f"-{name}.png")}" alt="image" style="width: 300px;">
|
||||
<br/>
|
||||
<img src="{path.format(id + f"-({index})-mask.png")}" alt="image" style="width: 300px;">
|
||||
<p style="word-wrap: break-word; max-height: 300px; max-width: 300px; overflow: auto;"> {element_text} </p>
|
||||
</div>
|
||||
""")
|
||||
print(f"""
|
||||
<div class="column">
|
||||
<img src="{path.format(id + f"-({len(elements) - 1}).png")}" alt="image" style="width: 300px;">
|
||||
<br/>
|
||||
<p style="word-wrap: break-word; max-height: 300px; overflow: auto;"> Background </p>
|
||||
</div>
|
||||
""")
|
||||
print('</div>')
|
||||
|
||||
print("""
|
||||
</body>
|
||||
</html>
|
||||
""")
|
||||
22
scripts/jobgen.py
Normal file
22
scripts/jobgen.py
Normal file
File diff suppressed because one or more lines are too long
174
scripts/post_processing.py
Normal file
174
scripts/post_processing.py
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
import logging
|
||||
import asyncio
|
||||
import time
|
||||
from collections import Counter
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from azure.storage.blob.aio import BlobServiceClient, download_blob_from_url
|
||||
from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorCursor, AsyncIOMotorCollection, AsyncIOMotorDatabase
|
||||
|
||||
AZUREBLOB_SAS_URL = "https://internblob.blob.core.windows.net/v-lixinyang/?sp=racwdli&st=2023-09-17T15:37:58Z&se=2023-12-31T23:37:58Z&spr=https&sv=2022-11-02&sr=c&sig=u%2FPbZ4fNttAPeLj0NEEpX0eIgFcjhot%2Bmy3iGd%2BCmxk%3D"
|
||||
CONTAINER = "canva-render-10.19"
|
||||
MONGODB_URI = "mongodb://localhost:27017/canva"
|
||||
|
||||
class BlobAsync(object):
|
||||
|
||||
async def readall(self, blob):
|
||||
blob_service_client = BlobServiceClient(AZUREBLOB_SAS_URL)
|
||||
async with blob_service_client:
|
||||
container_client = blob_service_client.get_container_client(CONTAINER)
|
||||
# async for bname in container_client.list_blob_names():
|
||||
# print(bname)
|
||||
blob_client = container_client.get_blob_client(blob)
|
||||
|
||||
if not await blob_client.exists():
|
||||
return None
|
||||
stream = await blob_client.download_blob()
|
||||
|
||||
return stream.readall()
|
||||
|
||||
async def open_image(self, blob: str):
|
||||
async with BlobServiceClient(AZUREBLOB_SAS_URL) as blob_service_client:
|
||||
container_client = blob_service_client.get_container_client(CONTAINER)
|
||||
# async for bname in container_client.list_blob_names():
|
||||
# print(bname)
|
||||
blob_client = container_client.get_blob_client(blob)
|
||||
|
||||
if not await blob_client.exists():
|
||||
return None
|
||||
stream = await blob_client.download_blob()
|
||||
|
||||
buf = np.frombuffer(await stream.readall(), dtype=np.uint8)
|
||||
image = cv2.imdecode(buf, cv2.IMREAD_COLOR)
|
||||
|
||||
await blob_client.close()
|
||||
await container_client.close()
|
||||
|
||||
return image
|
||||
|
||||
async def upload_image(self, blob, image):
|
||||
async with BlobServiceClient(AZUREBLOB_SAS_URL) as blob_service_client:
|
||||
# Instantiate a new ContainerClient
|
||||
container_client = blob_service_client.get_container_client(CONTAINER)
|
||||
blob_client = container_client.get_blob_client(blob)
|
||||
is_success, buffer = cv2.imencode('.png', image)
|
||||
|
||||
await blob_client.upload_blob(data=buffer.tobytes(), overwrite=True)
|
||||
await blob_client.close()
|
||||
await container_client.close()
|
||||
|
||||
async def get_mask(img1, img2):
|
||||
"""Assume img1 and img2 are exactly the same, except text areas
|
||||
"""
|
||||
try:
|
||||
diff = cv2.absdiff(img1, img2)
|
||||
except:
|
||||
raise ValueError("img1 and img2 are not the same size")
|
||||
mask = cv2.cvtColor(diff, cv2.COLOR_RGBA2GRAY)
|
||||
thresh, binmask= cv2.threshold(mask, 10, 255, cv2.THRESH_BINARY)
|
||||
return thresh, binmask
|
||||
|
||||
async def filter_mask_size(mask, thresh=0.4):
|
||||
non_zero_pixels = cv2.countNonZero(mask)
|
||||
total_pixels = mask.shape[0] * mask.shape[1]
|
||||
if non_zero_pixels > total_pixels * thresh:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
mask_filtered_count = Counter()
|
||||
async def process_cdf(blob: BlobAsync, collection, cdf):
|
||||
folder = cdf["rendered_folder"]
|
||||
async with asyncio.TaskGroup() as g:
|
||||
task1 = g.create_task(blob.open_image(f"{folder}/t=true.png"))
|
||||
task2 = g.create_task(blob.open_image(f"{folder}/t=false.png"))
|
||||
img1, img2 = task1.result(), task2.result()
|
||||
if img1 is None and img2 is None:
|
||||
mask_filtered_count["not found"] += 1
|
||||
await collection.update_one({"_id": cdf["_id"]}, {"$set": {"last_mask_render": -1, "failed_reason": "not found both"}})
|
||||
return
|
||||
if img1 is None:
|
||||
mask_filtered_count["not found"] += 1
|
||||
await collection.update_one({"_id": cdf["_id"]}, {"$set": {"last_mask_render": -1, "failed_reason": "not found t=true"}})
|
||||
return
|
||||
if img2 is None:
|
||||
mask_filtered_count["not found"] += 1
|
||||
await collection.update_one({"_id": cdf["_id"]}, {"$set": {"last_mask_render": -1, "failed_reason": "not found t=false"}})
|
||||
return
|
||||
|
||||
try:
|
||||
binary_thresh, mask = await get_mask(img1, img2)
|
||||
except ValueError as e:
|
||||
await collection.update_one({"_id": cdf["_id"]}, {"$set": {"last_mask_render": -1, "failed_reason": "size not match"}})
|
||||
mask_filtered_count["size not match"] += 1
|
||||
return
|
||||
|
||||
mask_filters = [
|
||||
(filter_mask_size, "mask too small")
|
||||
]
|
||||
tasks = list()
|
||||
|
||||
async with asyncio.TaskGroup() as g:
|
||||
for f, reason in mask_filters:
|
||||
tasks.append((g.create_task(f(mask)), reason))
|
||||
for task, reason in tasks:
|
||||
if task.result():
|
||||
mask_filtered_count[reason] += 1
|
||||
await collection.update_one({"_id": cdf["_id"]}, {"$set": {"last_mask_render": -1, "failed_reason": reason}, "$unset": {"last_fetched": -1}})
|
||||
return
|
||||
await blob.upload_image(f"{folder}/mask.png", mask)
|
||||
await collection.update_one({"_id": cdf["_id"]}, {"$set": {"last_mask_render": time.time()}})
|
||||
mask_filtered_count["success"] += 1
|
||||
|
||||
async def main():
|
||||
client = AsyncIOMotorClient(MONGODB_URI)
|
||||
db = client.get_database("canva")
|
||||
collection = db["cdf"]
|
||||
|
||||
logger = logging.getLogger('azure.mgmt.resource')
|
||||
logger.setLevel(logging.WARNING)
|
||||
blob = BlobAsync()
|
||||
|
||||
cdf_cursor: AsyncIOMotorCursor = collection.find({
|
||||
'$or': [
|
||||
{ '$and': [
|
||||
{ 'rendered_folder': { '$exists': True } },
|
||||
{ 'last_fetched': { '$gt': 1697688216 } },
|
||||
{ 'last_fetched': { '$lt': time.time() - 600 } },
|
||||
{ 'last_mask_render': { '$exists': False }}
|
||||
]},
|
||||
{ '$and': [
|
||||
{ 'last_fetched': {'$gt': 1697998932}},
|
||||
{ 'last_mask_render': { '$not': { '$gt': 0 } } }
|
||||
]}
|
||||
]}, batch_size=400)
|
||||
cdf_list = await cdf_cursor.to_list(length=200)
|
||||
await cdf_cursor.close()
|
||||
while cdf_list is not []:
|
||||
async with asyncio.TaskGroup() as g:
|
||||
taskset = set()
|
||||
for cdf in cdf_list:
|
||||
taskset.add(
|
||||
g.create_task(process_cdf(blob, collection, cdf))
|
||||
)
|
||||
await asyncio.sleep(10)
|
||||
cdf_cursor: AsyncIOMotorCursor = collection.find({
|
||||
'$or': [
|
||||
{ '$and': [
|
||||
{ 'rendered_folder': { '$exists': True } },
|
||||
{ 'last_fetched': { '$gt': 1697688216 } },
|
||||
{ 'last_fetched': { '$lt': time.time() - 600 } },
|
||||
{ 'last_mask_render': { '$exists': False }}
|
||||
]},
|
||||
{ '$and': [
|
||||
{ 'last_fetched': {'$gt': 1697998932}},
|
||||
{ 'last_mask_render': { '$not': { '$gt': 0 } } }
|
||||
]}
|
||||
]}, batch_size=400)
|
||||
cdf_list = await cdf_cursor.to_list(length=200)
|
||||
await cdf_cursor.close()
|
||||
print(mask_filtered_count)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
33
scripts/text_mask_from_all_mask.py
Normal file
33
scripts/text_mask_from_all_mask.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import argparse
|
||||
import json
|
||||
from re import L
|
||||
import cv2
|
||||
import pprint
|
||||
import os.path as osp
|
||||
from cdf_parser import CdfParser
|
||||
from threading import Lock
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
output_f = open("cdfs_with_masks.json", "a+")
|
||||
output_lock = Lock()
|
||||
|
||||
def get_text_mask(line):
|
||||
data = json.loads(line)
|
||||
cdf_parser = CdfParser(data['content'], data['rendered_folder'])
|
||||
elements = cdf_parser.get_texts()
|
||||
data["text_layer"] = elements
|
||||
del data["content"]
|
||||
output_f.write(json.dumps(data) + "\n")
|
||||
|
||||
|
||||
def main(cdf_file):
|
||||
with open(cdf_file, 'r') as f:
|
||||
for line in f:
|
||||
with ThreadPoolExecutor(max_workers=24) as executor:
|
||||
executor.submit(get_text_mask, line)
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--cdf", type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
main(args.cdf)
|
||||
65
scripts/vis.py
Normal file
65
scripts/vis.py
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
import json
|
||||
from collections import Counter
|
||||
|
||||
SAS = "sp=racwdli&st=2023-09-17T15:37:58Z&se=2023-12-31T23:37:58Z&spr=https&sv=2022-11-02&sr=c&sig=u%2FPbZ4fNttAPeLj0NEEpX0eIgFcjhot%2Bmy3iGd%2BCmxk%3D" # e.g sp=r&st=...
|
||||
|
||||
with open("sample_cdfs.json", "r") as f:
|
||||
data = []
|
||||
for line in f:
|
||||
data.append(json.load(f))
|
||||
|
||||
print("""
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
.row {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
justify-content: start;
|
||||
align-items: center;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
.column {
|
||||
flex: 15%;
|
||||
padding: 0 16px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
""")
|
||||
|
||||
for index, item in enumerate(data):
|
||||
# NOTE: replace xxxxxxxxx with SAS!
|
||||
file = f"https://internblob.blob.core.windows.net/v-lixinyang/canva-render-11.30/{item['rendered_folder']}-()?{SAS}"
|
||||
if index % 1 == 0:
|
||||
print('<div class="row">')
|
||||
print(f"""
|
||||
<div class="column">
|
||||
<img src="{gt_file}" alt="image" style="max-width: 100%;">
|
||||
</div>
|
||||
<div class="column">
|
||||
<img src="{file}" alt="image" style="max-width: 100%;">
|
||||
</div>
|
||||
<div class="column">
|
||||
<img src="{if_file}" alt="image" style="max-width: 100%;">
|
||||
</div>
|
||||
<div class="column">
|
||||
<img src="{sdxl_file}" alt="image" style="max-width: 100%;">
|
||||
</div>
|
||||
""")
|
||||
if index % 1 == 0:
|
||||
print('</div>')
|
||||
print(f"""
|
||||
<div class="row">
|
||||
<p>Index: {index} <br>
|
||||
Category: {item["category"]} <br>
|
||||
{item["caption"]} <br>
|
||||
Tags: {item["tags"]} <br>
|
||||
Texts: {item["texts"]}</p>
|
||||
</div>
|
||||
""")
|
||||
|
||||
print("""
|
||||
</body>
|
||||
</html>
|
||||
""")
|
||||
70
serve.py
Normal file
70
serve.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import os
|
||||
from typing import Annotated
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, Query, Request
|
||||
from fastapi.responses import HTMLResponse, Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.templating import Jinja2Templates
|
||||
|
||||
from cdf_parser import CdfParser
|
||||
|
||||
from render import Renderer
|
||||
|
||||
def handle_pdb(sig, frame):
|
||||
import pdb
|
||||
pdb.Pdb().set_trace(frame)
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
templates = Jinja2Templates(directory="templates")
|
||||
|
||||
renderer: Renderer = Renderer(debug=True, concurrent_workers=8, storage_state="894799196@qq.com.json")
|
||||
@app.on_event(r"startup")
|
||||
def startup():
|
||||
# FIXME: This background task cannot catch an exception
|
||||
asyncio.gather(renderer.run())
|
||||
|
||||
@app.get("/", response_class=Response)
|
||||
async def render():
|
||||
with open("web-2-J7IzVHyi0Zc.cdf", "rb") as f:
|
||||
cdf = f.read()
|
||||
rendered_page = await renderer.render_page(cdf)
|
||||
if rendered_page is None:
|
||||
return Response(status_code=500)
|
||||
return Response(content=await renderer.render_page(cdf), media_type="image/png")
|
||||
app.mount("/file", StaticFiles(directory="."), name="file")
|
||||
|
||||
@app.get("/file", response_class=HTMLResponse)
|
||||
def list_files(request: Request):
|
||||
files = os.listdir("./")
|
||||
files_paths = sorted([f"/file/{f}" for f in files])
|
||||
# print(files_paths)
|
||||
return templates.TemplateResponse(
|
||||
"list_files.html", {"request": request, "files": files_paths}
|
||||
)
|
||||
|
||||
@app.get("/worker/{worker_id}")
|
||||
async def get_worker(worker_id: int):
|
||||
return Response(content=await renderer.worker_screenshot(worker_id), media_type="image/png")
|
||||
|
||||
@app.get("/worker_content/{worker_id}")
|
||||
async def get_content(worker_id: int):
|
||||
return Response(content=await renderer.worker_content(worker_id))
|
||||
|
||||
@app.post("/render")
|
||||
async def render_cdf(cdf_dict: dict):
|
||||
cdf = json.dumps(cdf_dict)
|
||||
rendered_page = await renderer.render_page(cdf)
|
||||
if rendered_page is None:
|
||||
return Response(status_code=500)
|
||||
return Response(content=rendered_page, media_type="image/png")
|
||||
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run("serve:app", host="0.0.0.0", port=8000, log_level="debug")
|
||||
Binary file not shown.
13
templates/list_files.html
Normal file
13
templates/list_files.html
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
<html>
|
||||
<head>
|
||||
<title>Files</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Files:</h1>
|
||||
<ul>
|
||||
{% for file in files %}
|
||||
<li><a href="{{file}}">{{file}}</a></li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</body>
|
||||
</html>
|
||||
Loading…
Reference in a new issue