Auto upload base on #214 (#215)

* Update aria2.conf

* Add files via upload

* Update Dockerfile

* #214 Refactor the logic and make some of the parameters to be env variables

* #214 Update readme regarding auto upload

* Minor readme change

Co-authored-by: Kevin Rose <61625731+lvtai-py@users.noreply.github.com>

APP_VERSION

@@ -1 +1 @@
-4.14.1
+4.15.0

Dockerfile

@@ -35,6 +35,10 @@ ENV XDG_CACHE_HOME=/app/.cache
 ENV RCLONE_CONFIG_BASE64=""
 ENV ENABLE_APP_CHECKER=true
 ENV CADDY_LOG_LEVEL=INFO
+ENV RCLONE_AUTO_UPLOAD_PROVIDER=
+ENV RCLONE_AUTO_UPLOAD_REMOTE_PATH=/downloads
+ENV RCLONE_AUTO_UPLOAD_FILE_MIN_SIZE=1K
+ENV RCLONE_AUTO_UPLOAD_FILE_MAX_SIZE=100G
 
 ADD install.sh aria2c.sh caddy.sh Procfile init.sh start.sh rclone.sh new-version-checker.sh APP_VERSION /app/
 ADD conf /app/conf

README.CN.md

@@ -2,6 +2,8 @@ Aria2 + AriaNg
 
 [English](https://github.com/wahyd4/aria2-ariang-docker/blob/master/README.md) | 简体中文
 
+> 🚨 注意：由于本人精力有限，现决定将主要精力放在更新功能，和英文版本文档。中文版本文档即日起暂停更新，欢迎志愿者对英文说明进行翻译或者赞助本项目，助力我投入更多时间。谢谢。（2022年4月19日）
+
 [![](https://images.microbadger.com/badges/image/wahyd4/aria2-ui.svg)](https://microbadger.com/images/wahyd4/aria2-ui "Get your own image badge on microbadger.com")
 [![Docker Pulls](https://img.shields.io/docker/pulls/wahyd4/aria2-ui.svg)](https://hub.docker.com/r/wahyd4/aria2-ui/)
 [![Github Build](https://github.com/wahyd4/aria2-ariang-docker/workflows/Docker%20Image%20CI/badge.svg)](https://github.com/wahyd4/aria2-ariang-docker/actions)

README.md

@@ -52,6 +52,7 @@ File Browser
   * Basic Auth
   * Support ARM CPUs as well, all supported CPU platforms can be found [here](https://cloud.docker.com/repository/docker/wahyd4/aria2-ui/tags)
   * Cloud Storage platforms synchronization
+  * Auto uploading files to 3rd party Cloud storage providers via Rclone after files been downloaded.
 
 ## Recommended versions
 
@@ -87,7 +88,9 @@ File Browser
   -e ARIA2_USER=user \
   -e ARIA2_PWD=password \
   -e ARIA2_EXTERNAL_PORT=443 \
+  -e CADDY_LOG_LEVEL=ERROR \
   -v /yourdata:/data \
+  -v /app/.cache:/app/.cache \
   -v /app/a.db:/app/filebrowser.db \
   -v /to_yoursslkeys/:/app/conf/key \
   -v <conf files folder>:/app/conf \
@@ -115,30 +118,38 @@ services:
 Then simply run `docker-compose up -d`, that's it!
 
 ### Supported Environment Variables
-
-  * `ENABLE_AUTH` whether to enable Basic auth
-  * `ENABLE_RCLONE` whether to disable Rclone, if you running this container offline or do not have stable connection to Github, please set to `false`
-  * `ARIA2_USER` Basic Auth username, Rclone GUI uses it as well.
-  * `ARIA2_PWD` Basic Auth password, Rclone GUI uses it as well.
-  * `ARIA2_EXTERNAL_PORT` The Aria2 port which exposed to public to access to
-  * `PUID` Bind Linux UID into container which means you can use non `root` user to manage downloaded files, default UID is `1000`
-  * `PGID` Bind Linux GID into container, default GID is 1000
-  * `RPC_SECRET` The Aria2 RPC secret token
-  * `DOMAIN` The domain you'd like to bind, when domain is a `https://` thing, then auto TLS feature will be enabled
-  * `RCLONE_CONFIG_BASE64` Inject and config Rclone through `base64` string, which is the only way to use Rclone on Heroku. Please use `cat /app/conf/rclone.conf | base64` or any base64 online tools such as [this](https://www.base64encode.org/) to encode your `rclone.conf` as bse64 string. Note, you need to set `ENABLE_RCLONE` to true as well.
-  * `ENABLE_APP_CHECKER`, by default it's set to `true` to check if any new docker image version release on daily basis, which can help you get notification when new features released as well as some security vulnerabilities get fixed. You can set it to `false` to disable this feature. Note: you still need to manually pull the new image version and re run the docker container to complete upgrading.
+| ENV | Description|
+|:---|:---|
+| `ENABLE_AUTH` | Whether to enable Basic auth |
+| `ENABLE_RCLONE` | Whether to disable Rclone, if you running this container offline or do not have stable connection to Github, please set to `false`
+| `ARIA2_USER` | Basic Auth username, Rclone GUI uses it as well. |
+| `ARIA2_PWD` | Basic Auth password, Rclone GUI uses it as well. |
+| `ARIA2_EXTERNAL_PORT` | The Aria2 port which exposed to public to access to |
+| `PUID` | Bind Linux UID into container which means you can use non `root` user to manage downloaded files, default UID is `1000` |
+| `PGID` | Bind Linux GID into container, default GID is 1000 |
+| `RPC_SECRET` | The Aria2 RPC secret token |
+| `DOMAIN` | The domain you'd like to bind, when domain is a `https://` thing, then auto TLS feature will be enabled |
+| `RCLONE_CONFIG_BASE64` | Inject and config Rclone through `base64` string, which is the only way to use Rclone on Heroku. Please use `cat /app/conf/rclone.conf \| base64` or any base64 online tools such as [this](https://www.base64encode.org/) to encode your `rclone.conf` as bse64 string. Note, you need to set `ENABLE_RCLONE` to true as well. |
+| `ENABLE_APP_CHECKER` | By default it's set to `true` to check if any new docker image version release on daily basis, which can help you get notification when new features released as well as some security vulnerabilities get fixed. You can set it to `false` to disable this feature. Note: you still need to manually pull the new image version and re run the docker container to complete upgrading. |
+| CADDY_LOG_LEVEL | For specifying the log level of Caddy, set it to `ERROR` to reduce logs. Default: `INFO` |
+| RCLONE_AUTO_UPLOAD_PROVIDER | The Rclone remote storage provider name, which can be found under `Rclone -> Configs`, default `""`, which means auto upload is disabled. When the value is not empty, then the files will be attempted to be uploaded.|
+| RCLONE_AUTO_UPLOAD_REMOTE_PATH | The file folder in remote cloud storage provider, default `/downloads`|
+| RCLONE_AUTO_UPLOAD_FILE_MIN_SIZE | Set the minimum file size of auto uploader, files smaller than it won't be uploaded, default `1K`  |
+| RCLONE_AUTO_UPLOAD_FILE_MAX_SIZE | Set the limit of the Max file can be uploaded to 3rd party storage provider, default `100G`. |
 
 
 ### Supported Volumes
-  * `/data` The folder contains all the files you download.
-  * `/app/conf/key` The folder which stores Aria2 SSL `certificate` and `key` file. `Notice`: The certificate file should be named `aria2.crt` and the key file should be named `aria2.key`
-  * `/app/conf` The Aria2 configuration and file session folder. Make sure you have `aria2.conf` and `aria2.session` file. For the first time `aria2.session` just need to be a empty file can be appended. You can also user the templates for these two file in the `conf` folder of this project. Please put your `rclone.conf` in this folder as well if you'd mount it to Rclone. So all the config files supported in this folder are:
-    * aria2.conf
-    * aria2.session
-    * rclone.conf
 
-    **Warning: if you don't mount `/app/conf`, whenever the container restarts, you'll lose your downloading progress.**
-  * `/app/filebrowser.db` File Browser settings database, make sure you make a empty file first on your host.
+| Mountable folder | Description|
+|:---|:---|
+| `/data` | The folder contains all the files you download. |
+| `/app/conf/key` | The folder which stores Aria2 SSL `certificate` and `key` |file. `Notice`: The certificate file should be named `aria2.crt` and the key file should be named `aria2.key` |
+| `/app/conf` | The Aria2 configuration and file session folder. Make sure you have `aria2.conf` and `aria2.session` file. For the first time `aria2.session` just need to be a empty file can be appended. You can also user the templates for these two file in the `conf` folder of this project. Please put your `rclone.conf` in this folder as well if you'd mount it to Rclone. So all the config files supported in this folder are: `aria2.conf`, `aria2.session`, `rclone.conf`. 🚨Warning🚨: if you don't mount `/app/conf`, whenever the container restarts, you'll lose your downloading progress. |
+|`/app/conf/aria2.conf` | See description above👆🏼 |
+|`/app/conf/aria2.session` | See description above👆🏼 |
+|`/app/conf/rclone.conf` | See description above👆🏼 |
+| `/app/filebrowser.db` | File Browser settings database, make sure you make a empty file first on your host. |
+| `/app/.cache` | The folder for storing rclone caches and [aria2 DHT files](https://aria2.github.io/manual/en/html/aria2c.html#cmdoption-dht-file-path) |
 
 ## Auto HTTPS enabling
 

conf/aria2.conf

@@ -69,7 +69,8 @@ save-session-interval=10
 bt-tracker=http://1337.abcvg.info:80/announce,http://nyaa.tracker.wf:7777/announce,http://opentracker.xyz:80/announce,http://rt.tace.ru:80/announce,http://share.camoe.cn:8080/announce,http://t.nyaatracker.com:80/announce,http://torrentsmd.com:8080/announce,http://tr.cili001.com:8070/announce,http://tracker-cdn.moeking.me:2095/announce,http://tracker.anirena.com:80/announce,http://tracker.anirena.com:80/b16a15d9a238d1f59178d3614b857290/announce,http://tracker.anonwebz.xyz:8080/announce,http://tracker.bt4g.com:2095/announce,http://tracker.files.fm:6969/announce,http://tracker.gbitt.info:80/announce,http://tracker.ipv6tracker.ru:80/announce,http://tracker.noobsubs.net:80/announce,http://tracker.shittyurl.org:80/announce,http://tracker.tfile.co:80/announce,http://tracker.tfile.me:80/announce,http://tracker.trackerfix.com:80/announce,http://vps02.net.orel.ru:80/announce,https://1337.abcvg.info:443/announce,https://tr.torland.ga:443/announce,https://tracker.coalition.space:443/announce,https://tracker.gbitt.info:443/announce,https://tracker.lilithraws.cf:443/announce,https://tracker.nanoha.org:443/announce,https://tracker.nitrix.me:443/announce,https://tracker.parrotsec.org:443/announce,https://tracker.shittyurl.org:443/announce,https://tracker.tamersunion.org:443/announce,https://w.wwwww.wtf:443/announce,udp://6rt.tace.ru:80/announce,udp://9.rarbg.com:2830/announce,udp://9.rarbg.me:2710/announce,udp://9.rarbg.to:2710/announce,udp://app.icon256.com:8000/announce,udp://bubu.mapfactor.com:6969/announce,udp://discord.heihachi.pw:6969/announce,udp://exodus.desync.com:6969/announce,udp://explodie.org:6969/announce,udp://fe.dealclub.de:6969/announce,udp://inferno.demonoid.is:3391/announce,udp://ipv6.tracker.zerobytes.xyz:16661/announce,udp://mail.realliferpg.de:6969/announce,udp://movies.zsw.ca:6969/announce,udp://mts.tvbit.co:6969/announce,udp://open.demonii.com:1337/announce,udp://open.stealth.si:80/announce,udp://opentor.org:2710/announce,udp://opentracker.i2p.rocks:6969/announce,udp://p4p.arenabg.ch:1337/announce,udp://p4p.arenabg.com:1337/announce,udp://pow7.com:80/announce,udp://retracker.hotplug.ru:2710/announce,udp://retracker.lanta-net.ru:2710/announce,udp://retracker.netbynet.ru:2710/announce,udp://retracker01-msk-virt.corbina.net:80/announce,udp://t1.leech.ie:1337/announce,udp://t2.leech.ie:1337/announce,udp://thetracker.org:80/announce,udp://torrentclub.online:54123/announce,udp://tracker-de.ololosh.space:6969/announce,udp://tracker.0x.tf:6969/announce,udp://tracker.altrosky.nl:6969/announce,udp://tracker.army:6969/announce,udp://tracker.beeimg.com:6969/announce,udp://tracker.birkenwald.de:6969/announce,udp://tracker.blacksparrowmedia.net:6969/announce,udp://tracker.breizh.pm:6969/announce,udp://tracker.ccp.ovh:6969/announce,udp://tracker.cyberia.is:6969/announce,udp://tracker.dler.com:6969/announce,udp://tracker.internetwarriors.net:1337/announce,udp://tracker.moeking.me:6969/announce,udp://tracker.nrx.me:6969/announce,udp://tracker.ololosh.space:6969/announce,udp://tracker.openbittorrent.com:6969/announce,udp://tracker.opentrackr.org:1337/announce,udp://tracker.shkinev.me:6969/announce,udp://tracker.theoks.net:6969/announce,udp://tracker.tiny-vps.com:6969/announce,udp://tracker.torrent.eu.org:451/announce,udp://tracker.uw0.xyz:6969/announce,udp://tracker.zerobytes.xyz:1337/announce,udp://tracker0.ufibox.com:6969/announce,udp://tracker1.bt.moack.co.kr:80/announce,udp://tracker1.itzmx.com:8080/announce,udp://tracker2.dler.com:80/announce,udp://tracker2.dler.org:80/announce,udp://tracker2.itzmx.com:6961/announce,udp://tracker3.itzmx.com:6961/announce,udp://tracker4.itzmx.com:2710/announce,udp://u.wwwww.wtf:1/announce,udp://valakas.rollo.dnsabr.com:2710/announce,udp://vibe.community:6969/announce,udp://wassermann.online:6969/announce,udp://www.torrent.eu.org:451/announce
 
 # The script to be run when download completes
-on-download-complete=/app/conf/aria2-sample-hook.sh
+#on-download-complete=/app/conf/aria2-sample-hook.sh
+on-download-complete=/app/conf/auto-upload.sh
 
 # The script to be run when download fails
 on-download-error=/app/conf/aria2-sample-hook.sh

conf/auto-upload.sh

@@ -0,0 +1,42 @@
+#!/bin/bash -eu
+RCLONE_AUTO_UPLOAD_PROVIDER=google-drive
+if [[ $RCLONE_AUTO_UPLOAD_PROVIDER == "" ]]; then
+    echo "[INFO] $(date -u +'%Y-%m-%dT%H:%M:%SZ') Auto upload isn't enabled"
+    exit 0
+fi
+
+if [ $2 -eq 0 ]; then
+    echo "[WARN] $(date -u +'%Y-%m-%dT%H:%M:%SZ') No files detected"
+    exit 0
+fi
+
+echo "[INFO] $(date -u +'%Y-%m-%dT%H:%M:%SZ') Start to upload files with: Files Count: [$2], Files Path: [$3]"
+
+path=$3                                    # File full path
+downloadpath='/data'                       # Aria2 downloading folder
+provider_name=$RCLONE_AUTO_UPLOAD_PROVIDER # RClone remote provider
+folder=$RCLONE_AUTO_UPLOAD_REMOTE_PATH     # Foler path in Rclone remote provider
+MinSize=$RCLONE_AUTO_UPLOAD_FILE_MIN_SIZE  # Set mininum file size to be uploaded, files smaller than this will be deleted without uploading
+MaxSize=$RCLONE_AUTO_UPLOAD_FILE_MAX_SIZE  # Set the Max file size to be uploaded
+
+while true; do #提取下载文件根路径，如把/data/a/b/c/d.jpg变成/data/a
+    file_path=$path
+    path=${path%/*}
+    if [ "$path" == "" ]; then #不在下载文件夹
+        exit 0
+    elif [ "$path" == "$downloadpath" ] && [ $2 -eq 1 ]; then # To handle single file
+        rclone move "$file_path" ${provider_name}:${folder}/
+        echo "[INFO] $(date -u +'%Y-%m-%dT%H:%M:%SZ') Successfully uploaded file $file_path"
+        exit 0
+    elif [ "$path" == "$downloadpath" ]; then # To handle folder
+        while [[ "$(ls -A "$file_path/")" != "" ]]; do
+            rclone move "$file_path"/ ${provider_name}:${folder}/"${file_path##*/}"/ --delete-empty-src-dirs --min-size $MinSize --max-size $MaxSize
+
+            echo "[INFO] $(date -u +'%Y-%m-%dT%H:%M:%SZ') Successfully uploaded file $file_path"
+
+            rclone delete -v "$file_path" --max-size $MinSize # Delete files smaller than min file.
+        done
+        rm -rf "$file_path/"
+        exit 0
+    fi
+done