Pre-Compiled and Upload #7
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Pre-Compiled and Upload | |
| on: | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| Windows: | |
| runs-on: windows-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.10"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| .\run-install.bat | |
| .\env\python.exe core.py "prerequisites" | |
| - name: Clean up unnecessary files | |
| run: | | |
| Remove-Item -Path 'LICENSE', 'run-tensorboard.sh', 'run-install.sh', 'run-applio.sh', 'run-install.bat', 'requirements.txt', '.gitignore', '.github', '.git' -Recurse -Force -ErrorAction SilentlyContinue | |
| Get-ChildItem -Path . -Include __pycache__ -Recurse -Directory | Remove-Item -Force -Recurse -ErrorAction SilentlyContinue | |
| - name: Read version from config.json | |
| id: get-version | |
| run: | | |
| $version = (Get-Content -Path './assets/config.json' | ConvertFrom-Json).version | |
| echo "version=$version" >> $env:GITHUB_OUTPUT | |
| - name: Create ZIP file | |
| run: | | |
| $version = '${{ steps.get-version.outputs.version }}' | |
| Compress-Archive -Path 'D:\a\Applio\Applio\*' -DestinationPath "D:\a\Applio\ApplioV$version.zip" | |
| - name: Upload Windows Compiled | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| version: ${{ steps.get-version.outputs.version }} | |
| run: | | |
| .\env\python.exe -c " | |
| import os | |
| from huggingface_hub import login, upload_file | |
| token = os.getenv('HF_TOKEN') | |
| version = os.getenv('version') | |
| login(token) | |
| repo_id = 'IAHispano/Applio' | |
| repo_file_path = f'D:/a/Applio/ApplioV{version}.zip' | |
| upload_file( | |
| commit_message=f'{version}', | |
| path_or_fileobj=repo_file_path, | |
| path_in_repo=f'Compiled/Windows/ApplioV{version}.zip', | |
| repo_id=repo_id, | |
| token=token) | |
| " | |
| Linux: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.10.12"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Free up space on runner | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Read version from config.json | |
| id: get-version | |
| run: | | |
| version=$(jq -r '.version' assets/config.json) | |
| echo "version=$version" >> $GITHUB_OUTPUT | |
| - name: Create virtual environment | |
| run: | | |
| pip install huggingface_hub | |
| python -m venv .venv | |
| . .venv/bin/activate | |
| python -m ensurepip | |
| python -m pip install "pip<24.1" | |
| python -m pip install -r requirements.txt | |
| python -m pip install torch==2.3.1 torchvision torchaudio --upgrade --index-url https://download.pytorch.org/whl/cu121 | |
| python core.py "prerequisites" | |
| find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete | |
| deactivate | |
| rm -f .venv/bin/Activate.ps1 .venv/bin/activate .venv/bin/activate.csh .venv/bin/activate.fish .venv/bin/pip .venv/bin/pip3 .venv/bin/pip3.10 .venv/bin/python .venv/bin/python3 .venv/bin/python3.10 | |
| - name: Clean up unnecessary files | |
| run: | | |
| rm -f LICENSE run-tensorboard.bat run-install.bat run-applio.bat run-install.sh requirements.txt .gitignore | |
| rm -rf .github .git | |
| - name: Create ZIP file | |
| run: | | |
| zip -9 -r ApplioV${{ steps.get-version.outputs.version }}.zip . | |
| - name: Upload Linux Compiled | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| version: ${{ steps.get-version.outputs.version }} | |
| run: | | |
| python -c " | |
| import os | |
| from huggingface_hub import login, upload_file | |
| token = os.getenv('HF_TOKEN') | |
| version = os.getenv('version') | |
| login(token) | |
| repo_id = 'IAHispano/Applio' | |
| repo_file_path = f'ApplioV{version}.zip' | |
| upload_file( | |
| commit_message=f'{version}', | |
| path_or_fileobj=repo_file_path, | |
| path_in_repo=f'Compiled/Linux/ApplioV{version}.zip', | |
| repo_id=repo_id, | |
| token=token) | |
| " | |
| Kaggle: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.10.12"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Free up space on runner | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Read version from config.json | |
| id: get-version | |
| run: | | |
| version=$(jq -r '.version' assets/config.json) | |
| echo "version=$version" >> $GITHUB_OUTPUT | |
| - name: Install uv and create virtual environment | |
| run: | | |
| sudo apt-get install -y pigz | |
| sudo apt-get install -y pv | |
| pip install uv huggingface_hub | |
| uv venv .venv --python 3.10.12 | |
| uv pip install -r requirements.txt | |
| uv pip install https://github.com/IAHispano/imjoy-elfinder.git | |
| uv pip uninstall torch torchvision torchaudio | |
| . .venv/bin/activate | |
| python -m ensurepip | |
| python -m pip install "pip<24.1" | |
| python -m pip install torch==2.3.1 torchvision torchaudio --upgrade --index-url https://download.pytorch.org/whl/cu121 | |
| deactivate | |
| wget https://huggingface.co/IAHispano/Applio/resolve/main/Enviroment/Kaggle/bin_kaggle.tar.gz?download=true -O bin_kaggle.tar.gz | |
| tar -xzf bin_kaggle.tar.gz | |
| cp -r bin/* .venv/bin/ | |
| rm -rf bin | |
| rm bin_kaggle.tar.gz | |
| rm -f .venv/pyvenv.cfg .venv/bin/activate .venv/bin/activate.bat .venv/bin/activate.csh .venv/bin/activate.fish .venv/bin/activate.nu .venv/bin/activate.ps1 .venv/bin/activate_this.py .venv/bin/deactivate.bat .venv/bin/pydoc.bat .venv/bin/python .venv/bin/python3 .venv/bin/python3.10 | |
| - name: Create TAR Env | |
| run: | | |
| tar --use-compress-program="pigz --best --recursive | pv" -cvf KaggleV2.tar.gz .venv | |
| - name: Upload Kaggle Env | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| version: ${{ steps.get-version.outputs.version }} | |
| run: | | |
| python -c " | |
| import os | |
| from huggingface_hub import login, upload_file | |
| token = os.getenv('HF_TOKEN') | |
| version = os.getenv('version') | |
| login(token) | |
| repo_id = 'IAHispano/Applio' | |
| repo_file_path = 'KaggleV2.tar.gz' | |
| upload_file( | |
| commit_message=f'{version}', | |
| path_or_fileobj=repo_file_path, | |
| path_in_repo=f'Enviroment/Kaggle/{repo_file_path}', | |
| repo_id=repo_id, | |
| token=token) | |
| " | |
| Colab: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.10.12"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Download and extract pre-packaged dependencies | |
| run: | | |
| wget https://huggingface.co/IAHispano/Applio/resolve/main/Enviroment/Colab/tmp_packages.tar.gz?download=true -O prepackaged.tar.gz | |
| tar -xzf prepackaged.tar.gz -C /opt/hostedtoolcache/Python/3.10.12/x64/lib/python3.10/site-packages/ | |
| - name: Read version from config.json | |
| id: get-version | |
| run: | | |
| version=$(jq -r '.version' assets/config.json) | |
| echo "version=$version" >> $GITHUB_OUTPUT | |
| - name: Record initial state of site-packages | |
| run: | | |
| python -c " | |
| import csv | |
| import os | |
| def scan_and_write(base_path, output_file): | |
| with open(output_file, 'w', newline='') as f: | |
| writer = csv.writer(f) | |
| for dirpath, dirs, files in os.walk(base_path): | |
| for filename in files: | |
| fname = os.path.join(dirpath, filename) | |
| try: | |
| mtime = os.path.getmtime(fname) | |
| writer.writerow([fname, mtime]) | |
| except Exception as e: | |
| print(f'Skipping irrelevant nonexistent file {fname}: {str(e)}') | |
| print(f'Finished recording filesystem timestamps to {output_file}.') | |
| scan_and_write(\"/opt/hostedtoolcache/Python/3.10.12/x64\", \"./usr_files.csv\")" | |
| - name: Install dependencies | |
| run: | | |
| pip install -r requirements.txt | |
| pip install torch==2.3.1 torchvision torchaudio --upgrade --index-url https://download.pytorch.org/whl/cu121 | |
| - name: Record final state of site-packages | |
| run: | | |
| python -c " | |
| import csv | |
| import os | |
| def scan_and_write(base_path, output_file): | |
| with open(output_file, 'w', newline='') as f: | |
| writer = csv.writer(f) | |
| for dirpath, dirs, files in os.walk(base_path): | |
| for filename in files: | |
| fname = os.path.join(dirpath, filename) | |
| try: | |
| mtime = os.path.getmtime(fname) | |
| writer.writerow([fname, mtime]) | |
| except Exception as e: | |
| print(f'Skipping irrelevant nonexistent file {fname}: {str(e)}') | |
| print(f'Finished recording filesystem timestamps to {output_file}.') | |
| scan_and_write(\"/opt/hostedtoolcache/Python/3.10.12/x64\", \"./usr_files_new.csv\")" | |
| - name: Create TAR file | |
| run: | | |
| python -c " | |
| import tarfile | |
| import os | |
| from tqdm import tqdm | |
| import csv | |
| def compare_files(old_file, new_file): | |
| old_files = {} | |
| new_files = {} | |
| with open(old_file, 'r') as f: | |
| reader = csv.reader(f) | |
| old_files = {rows[0]: rows[1] for rows in reader} | |
| with open(new_file, 'r') as f: | |
| reader = csv.reader(f) | |
| new_files = {rows[0]: rows[1] for rows in reader} | |
| removed_files = old_files.keys() - new_files.keys() | |
| added_files = new_files.keys() - old_files.keys() | |
| unchanged_files = old_files.keys() & new_files.keys() | |
| changed_files = {f for f in unchanged_files if old_files[f] != new_files[f]} | |
| for file in removed_files: | |
| print(f'File has been removed: {file}') | |
| for file in changed_files: | |
| print(f'File has been updated: {file}') | |
| return list(added_files) + list(changed_files) | |
| changed_files = compare_files('./usr_files.csv', './usr_files_new.csv') | |
| base_dir = '/opt/hostedtoolcache/Python/3.10.12/x64/' | |
| arc_base_dir = 'usr/local/' | |
| with tarfile.open('./Pnpur.tar.gz', 'w:gz') as new_tar: | |
| for file in tqdm(changed_files, desc='Compressing', unit='files'): | |
| relative_path = os.path.relpath(file, base_dir) | |
| if 'python3.10' in relative_path: | |
| if 'site-packages' not in relative_path: | |
| continue | |
| if 'site-packages' in relative_path: | |
| relative_path = relative_path.replace('site-packages', 'dist-packages') | |
| new_path = os.path.join(arc_base_dir, relative_path) | |
| new_tar.add(file, arcname=new_path) | |
| " | |
| - name: Upload Colab Env | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| version: ${{ steps.get-version.outputs.version }} | |
| run: | | |
| python -c " | |
| import os | |
| from huggingface_hub import login, upload_file | |
| token = os.getenv('HF_TOKEN') | |
| version = os.getenv('version') | |
| login(token) | |
| repo_id = 'IAHispano/Applio' | |
| repo_file_path = 'Pnpur.tar.gz' | |
| upload_file( | |
| commit_message=f'{version}', | |
| path_or_fileobj=repo_file_path, | |
| path_in_repo=f'Enviroment/Colab/{repo_file_path}', | |
| repo_id=repo_id, | |
| token=token) | |
| " |