diff --git a/Dockerfile b/Dockerfile
index 7e100f3..d6997c2 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -30,8 +30,8 @@ RUN \
       libffi-dev \
       openssl-dev \
       python3-dev \
+      py3-setuptools \
       py3-pip \
-      py3-wheel \
       libcdio \
       libcaca \
       libvpx \
@@ -67,7 +67,19 @@ RUN \
       Mopidy-SoundCloud \
   && echo "* Cleaning up" \
     && rm -rf /var/cache/apk/* \
-    && rm -rf /tmp/*
+    && rm -rf /root/.cache /root/.cargo \
+    && rm -rf /tmp/* \
+    && apk del \
+      build-base \
+      rust \
+      cargo \
+      libxml2-dev \
+      libxslt-dev \
+      libffi-dev \
+      openssl-dev \
+      python3-dev \
+      py3-pip
+
 
 # Do this in a separate layer, because building the last is slooow
 # This allows us to alter these parameters without rebuilding the last.