Merge pull request #12 from albarji/feature/style-swap

Major overhaul of the application and addition of faster algorithms

.gitignore

@@ -1 +1,4 @@
 **/__pycache__/
+*.pyc
+.idea
+.coverage

Dockerfile

@@ -18,40 +18,49 @@ RUN git clone https://github.com/torch/distro.git /root/torch --recursive && cd
 RUN cd /root/torch && ./install.sh
 RUN ln -s /root/torch/install/bin/* /usr/local/bin
 
-# Torch environment variables
-ENV LUA_PATH='/root/.luarocks/share/lua/5.1/?.lua;/root/.luarocks/share/lua/5.1/?/init.lua;/root/torch/install/share/lua/5.1/?.lua;/root/torch/install/share/lua/5.1/?/init.lua;./?.lua;/root/torch/install/share/luajit-2.1.0-beta1/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/?/init.lua'
-ENV LUA_CPATH='/root/.luarocks/lib/lua/5.1/?.so;/root/torch/install/lib/lua/5.1/?.so;./?.so;/usr/local/lib/lua/5.1/?.so;/usr/local/lib/lua/5.1/loadall.so'
-ENV PATH=/root/torch/install/bin:$PATH
-ENV LD_LIBRARY_PATH=/root/torch/install/lib:$LD_LIBRARY_PATH
-ENV DYLD_LIBRARY_PATH=/root/torch/install/lib:$DYLD_LIBRARY_PATH
-ENV LUA_CPATH='/root/torch/install/lib/?.so;'$LUA_CPATH
+# Install additional necessary torch dependencies
+RUN luarocks install loadcaffe && \
+    luarocks install autograd
 
-# Install loadcaffe and other torch dependencies
-RUN luarocks install loadcaffe
+# Install Python miniconda3 + requirements
+ENV MINICONDA_HOME="/opt/miniconda"
+ENV PATH="${MINICONDA_HOME}/bin:${PATH}"
+RUN curl -o Miniconda3-latest-Linux-x86_64.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+  && chmod +x Miniconda3-latest-Linux-x86_64.sh \
+  && ./Miniconda3-latest-Linux-x86_64.sh -b -p "${MINICONDA_HOME}" \
+&& rm Miniconda3-latest-Linux-x86_64.sh
 
 # Clone neural-style app
-WORKDIR /
+WORKDIR /app
 RUN set -ex && \
 	wget --no-check-certificate https://github.com/jcjohnson/neural-style/archive/master.tar.gz && \
 	tar -xvzf master.tar.gz && \
     mv neural-style-master neural-style && \
 	rm master.tar.gz
 
-# Download precomputed network weights
-WORKDIR neural-style
+# Download precomputed VGG network weights
+WORKDIR /app/neural-style
 RUN bash models/download_models.sh
-RUN mkdir /models
-
-# Copy wrapper scripts
-COPY ["/scripts/variants.sh", "/scripts/neural-style.sh", "/neural-style/"]
 
 # Add neural-style to path
-ENV PATH /neural-style:$PATH
+ENV PATH /app/neural-style:$PATH
 
-# Prepare folder for mounting images and workplaces
-WORKDIR /images
-VOLUME ["/images"]
+# Clone style-swap app
+WORKDIR /app
+RUN set -ex && \
+	wget --no-check-certificate https://github.com/rtqichen/style-swap/archive/master.tar.gz && \
+	tar -xvzf master.tar.gz && \
+    mv style-swap-master style-swap && \
+	rm master.tar.gz
+# Link precomputed VGG network weights
+RUN rm -rf /app/style-swap/models
+RUN ln -s /app/neural-style/models /app/style-swap/models
+# Add precomputed inverse network model
+ADD models/dec-tconv-sigmoid.t7 /app/style-swap/models/dec-tconv-sigmoid.t7
+
+# Copy wrapper scripts
+COPY ["entrypoint.py" ,"/app/entrypoint/"]
+COPY ["/neuralstyle/*.py", "/app/entrypoint/neuralstyle/"]
 
-ENTRYPOINT ["neural-style.sh"]
-CMD ["-backend", "cudnn", "-cudnn_autotune"]
+ENTRYPOINT ["python", "/app/entrypoint/entrypoint.py"]
 

Makefile

@@ -1,10 +1,21 @@
-.PHONY: build clean
+.PHONY: all build clean build_tests tests clean_tests
 IMGNAME=albarji/neural-style
 IMGTAG=latest
+TESTIMGNAME=$(IMGNAME)-tests
 
 build:
 	nvidia-docker build -t $(IMGNAME):$(IMGTAG) .
 
 clean:
 	docker rmi $(IMGNAME):$(IMGTAG)
 
+build_tests:
+	nvidia-docker build -t $(TESTIMGNAME) tests
+
+tests:
+	nvidia-docker run --rm -it $(TESTIMGNAME)
+
+clean_tests:
+	docker rmi $(TESTIMGNAME)
+
+all: build build_tests tests

README.md

@@ -13,7 +13,8 @@
 ![Stylized Docker](./doc/docker_seated-nude_sw375_ss1.png)
 ![Stylized Docker](./doc/docker_starryNight_sw1500_ss1.png)
 
-A dockerized version of the [neural style algorithm by jcjohnson](https://github.com/jcjohnson/neural-style). [nvidia-docker](https://github.com/NVIDIA/nvidia-docker) is used to make use of GPU hardware.
+A dockerized version of neural style transfer algorithms.
+[nvidia-docker](https://github.com/NVIDIA/nvidia-docker) is used to make use of GPU hardware.
 
 ## Install
 
@@ -33,50 +34,87 @@ or build the image locally with
 
 	make
 
-## Simple use
+### Usage
 
-Just run
+This docker container operates by receiving images through a volume to be mounted at the **/images** directory.
+For instance, to apply a **style** image *somestyle.png* onto a **content** image *somecontent.png* located at the 
+current directory, run: 
 
-	bash scripts/fake-it.sh
+    nvidia-docker run --rm -v $(pwd):/images albarji/neural-style --content somecontent.png --style somestyle.png
 
-This applies a blend of content and style with some default parameters. Both content and style images must be present in the "contents" and "styles" folders, respectively.
+All paths referenced in the arguments are regarded as relative to the /images folder within the container. So in case
+of having a local structure such as
 
-Example: to draw the Golden Gate bridge the style of Van Gogh's Starry Night, type
+    contents/
+        docker.png
+        whatever.jpg
+    styles/
+        picasso.png
+        vangogh.png
+        
+applying the *vangogh.png* style to the *docker.png* image amounts to
 
-	bash scripts/fake-it.sh goldengate.jpg vangogh.jpg
+    nvidia-docker run --rm -v $(pwd):/images albarji/neural-style --content contents/docker.png --style styles/vangogh.png
+
+You can provide several content and style images, in which case all cross-combinations will be generated.
 
-## Advanced use
+    nvidia-docker run --rm -v $(pwd):/images albarji/neural-style --content contents/docker.png contents/whatever.jpg --style styles/vangogh.png styles/picasso.png
 
-### Generating variants
+### Fine tuning the results
 
-Running the command script
+Better results can be attained by modifying some of the transfer parameters.
 
-	bash scripts/variants.sh
+#### Algorithm
 
-will generate several variants of the same image blends, for different neural-style parameters that work well in general. This is useful for producing several versions of the same blend and afterwards hand-picking the best one. Run this command with the -h option to obtain usage help.
+The --alg parameter allows changing the neural style transfer algorithm to use.
 
-For example, to generate different variants of Docker logo + Starry Night:
+* **gatys**: highly detailed transfer, slow processing times (default)
+* **chen-schmidt**: fast patch-based style transfer
+* **chen-schmidt-inverse**: even faster aproximation to chen-schmidt through the use of an inverse network
 
-	bash scripts/variants.sh --contents contents/docker.png --styles styles/vangogh.jpg
+The following example illustrates kind of results to be expected by these different algorithms
 
-### Use as the neural-style command
+| Content image | Algorithm | Style image |
+| ------------- | --------- | ----------- |
+| ![Content](./doc/avila-walls.jpg) | Gatys ![Gatys](./doc/avila-walls_broca_gatys_ss1.0_sw10.0.jpg) | ![Style](./doc/broca.jpg) | 
+| ![Content](./doc/avila-walls.jpg) | Chen-Schmidt ![Chen-Schmidt](./doc/avila-walls_broca_chen-schmidt_ss1.0.jpg) | ![Style](./doc/broca.jpg) | 
+| ![Content](./doc/avila-walls.jpg) | Chen-Schmidt Inverse ![Chen-Schmidt Inverse](./doc/avila-walls_broca_chen-schmidt-inverse_ss1.0.jpg) | ![Style](./doc/broca.jpg) | 
 
-You can directly invoke the core neural-style algorithm by simply running a container of this image, for example:
+#### Output image size
 
-	nvidia-docker run --rm albarji/neural-style -h
+By defaul the output image will have the same size as the input content image, but a different target size can be
+specified through the --size parameter. For example, to produce a 512 image
 
-produces the usage help.
+    nvidia-docker run --rm -v $(pwd):/images albarji/neural-style --content contents/docker.png --style styles/vangogh.png --size 512
+
+Note the proportions of the image are maintained, therefore the value of the size parameter is understood as the width 
+of the target image, the height being scaled accordingly to keep proportion.  
 
-To apply the neural-style method on some host images, map the host folder with such images to the container /images folder through a volume such as
+#### Style weight
 
-	nvidia-docker run --rm -v $(pwd):/images albarji/neural-style -backend cudnn -cudnn_autotune -content_image content.png -style_image style.png
+Gatys algorithm allows to adjust the amount of style imposed over the content image, by means of the --sw parameter.
+By default a value of **10** is used, meaning the importance of the style is 10 times the importance of the content.
+Smaller weight values result in the transfer of colors, while higher values transfer textures and details of the style
 
-The container uses as work directory the /images folder, so the results will be readily available at the mounted host folder.
+If several weight values can be provided, all combinations will be generated. For instance, to generate the same
+style transfer with three different weights, use
 
-In order to take full advantage of the cudnn libraries (also included in the image) the options -backend cudnn -cudnn_autotune are always recommended.
+    nvidia-docker run --rm -v $(pwd):/images albarji/neural-style --content contents/docker.png --style styles/vangogh.png --sw 5 10 20
+
+#### Style scale
 
-As an example, let's redraw Docker's logo in the famous style of Van Gogh's Starry Night:
+If the transferred style results in too large or too small features, the scaling can be modified through the --ss 
+parameter. A value of **1** keeps the style at its original scale. Smaller values reduce the scale of the style,
+resulting in smaller style features in the output image. Conversely, larger values produce larger features. 
+Similarly to the style weight, several values can be provided
 
-	nvidia-docker run --rm -v $(pwd):/images albarji/neural-style -backend cudnn -cudnn_autotune -content_image contents/docker.png -style_image styles/vangogh.jpg
+    nvidia-docker run --rm -v $(pwd):/images albarji/neural-style --content contents/docker.png --style styles/vangogh.png --ss 0.75 1 1.25
+
+Warning: using a value larger than **1** will increasy the memory consumption. 
 
+## References
 
+* [Gatys et al method](https://arxiv.org/abs/1508.06576), [implementation by jcjohnson](https://github.com/jcjohnson/neural-style)
+* [Chen-Schmidt method](https://arxiv.org/pdf/1612.04337.pdf), [implementation](https://github.com/rtqichen/style-swap)
+* [A review on style transfer methods](https://arxiv.org/pdf/1705.04058.pdf)
+* [Neural-tiling method](https://github.com/ProGamerGov/Neural-Tile)

entrypoint.py

@@ -0,0 +1,103 @@
+# Main entrypoint script to the neural-style app
+import sys
+import traceback
+import logging
+from neuralstyle.algorithms import styletransfer
+from neuralstyle.utils import sublist
+
+logging.basicConfig(level=logging.INFO)
+LOGGER = logging.getLogger(__name__)
+
+HELP = """
+neural-style-docker: artistic style between images
+
+    --content CONTENT_IMAGES: file or files with the content images to use
+    --style STYLE_IMAGES: file or files with the styles to transfer
+    --output OUTPUT_FOLDER: name of the output folder in which to save results
+    --size SIZE: size of the output image. Default: content image size
+    --sw STYLE_WEIGHT (default 10): weight or list of weights of the style over the content, in range (0, inf)
+    --ss STYLE_SCALE (default 1.0): scaling or list of scaling factors for the style images
+    --alg ALGORITHM: style-transfer algorithm to use. Must be one of the following:
+        gatys                   Highly detailed transfer, slow processing times (default)
+        chen-schmidt            Fast patch-based style transfer
+        chen-schmidt-inverse    Even faster aproximation to chen-schmidt through the use of an inverse network
+    --tilesize TILE_SIZE: maximum size of each tile in the style transfer.
+        If your GPU runs out of memory you should try reducing this value. Default: 512
+
+    Additionally provided parameters are carried on to the underlying algorithm.
+    
+"""
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv
+    try:
+        # Default parameters
+        contents = []
+        styles = []
+        savefolder = "/images"
+        size = None
+        alg = "gatys"
+        weights = None
+        stylescales = None
+        otherparams = []
+
+        # Gather parameters
+        i = 1
+        while i < len(argv):
+            # References to inputs/outputs are re-referenced to the mounted /images directory
+            if argv[i] == "--content":
+                contents = ["/images/" + x for x in sublist(argv[i+1:], stopper="--")]
+                i += len(contents) + 1
+            elif argv[i] == "--style":
+                styles = ["/images/" + x for x in sublist(argv[i+1:], stopper="--")]
+                i += len(styles) + 1
+            # Other general parameters
+            elif argv[i] == "--output":
+                savefolder = argv[i+1]
+                i += 2
+            elif argv[i] == "--alg":
+                alg = argv[i+1]
+                i += 2
+            elif argv[i] == "--size":
+                size = int(argv[i+1])
+                i += 2
+            elif argv[i] == "--sw":
+                weights = [float(x) for x in sublist(argv[i+1:], stopper="--")]
+                i += len(weights) + 1
+            elif argv[i] == "--ss":
+                stylescales = [float(x) for x in sublist(argv[i+1:], stopper="--")]
+                i += len(stylescales) + 1
+            # Help
+            elif argv[i] == "--help":
+                print(HELP)
+                return 0
+            # Additional parameters will be passed on to the specific algorithms
+            else:
+                otherparams.append(argv[i])
+                i += 1
+
+        # Check parameters
+        if len(contents) == 0:
+            raise ValueError("At least one content image must be provided")
+        if len(styles) == 0:
+            raise ValueError("At least one style image must be provided")
+
+        LOGGER.info("Running neural style transfer with")
+        LOGGER.info("\tContents = %s" % str(contents))
+        LOGGER.info("\tStyle = %s" % str(styles))
+        LOGGER.info("\tAlgorithm = %s" % alg)
+        LOGGER.info("\tStyle weights = %s" % str(weights))
+        LOGGER.info("\tStyle scales = %s" % str(stylescales))
+        styletransfer(contents, styles, savefolder, size, alg, weights, stylescales, *otherparams)
+        return 1
+
+    except Exception:
+        print(HELP)
+        traceback.print_exc()
+        return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())