diff --git a/scripts/singularity/ExtensionCrawler.def b/scripts/singularity/ExtensionCrawler.def new file mode 100644 index 0000000..4307cb9 --- /dev/null +++ b/scripts/singularity/ExtensionCrawler.def @@ -0,0 +1,192 @@ +#!/bin/sh +# Copyright 2017 The University of Sheffield, UK +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +bootstrap:debootstrap +OSVersion: testing +MirrorURL: https://deb.debian.org/debian + +%labels +Maintainer The LogicalHacking Team (https://logicalhacking.com) + +%setup + +%post + +################################################################### +# Add Debian unstable as a secondary (lower priority) source +# and update the data base of available packages. +cat >> /etc/apt/sources.list << EOF +deb http://ftp.us.debian.org/debian unstable main +EOF + +cat > /etc/apt/preferences << EOF +Package: * +Pin: release a=testing +Pin-Priority: 900 + +Package: * +Pin: release a=unstable +Pin-Priority: 800 +EOF + +cat > /etc/apt/apt.conf.d/01norecommend << EOF +APT::Install-Recommends "0"; +APT::Install-Suggests "0"; +EOF + +chmod go+r /etc/apt/preferences +apt-get update +################################################################### + +################################################################### +# Add hook for apt that removes various files after installation +# that are not needed at runtime. +cat > /etc/apt/apt.conf.d/99-clean << EOF +DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; }; +APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; }; +Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache ""; +EOF +################################################################### + +################################################################### +# Configure locales +apt-get install -y locales +echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen +echo "en_GB.UTF-8 UTF-8" >> /etc/locale.gen +locale-gen +echo "LANG=en_US.UTF-8" > /etc/default/locale +################################################################### + +################################################################### +# Install the core dependencies (Python 3.6 or later) +# from the Debian Testing repository +apt-get install -y --no-install-recommends libpython3.7-dev python3-magic python3-minimal python3-pip python3-setuptools python3-mysqldb g++ git +apt-get clean +rm -rf /var/lib/apt/lists/* +################################################################### + +################################################################### +# Create /opt for local software (mainly cloned git repositories +# from logicalhacking.com +mkdir -p /opt +chmod 755 /opt +################################################################### + +################################################################### +# Add the Extension Crawler repository, for more details, visit +# https://git.logicalhacking.com/BrowserSecurity/ExtensionCrawler +cd /opt +git clone https://git.logicalhacking.com/BrowserSecurity/ExtensionCrawler.git +cd ExtensionCrawler +git checkout production +cd .. +pip3 install wheel # simhash needs wheel to build properly, still works without it though +pip3 install --system -e ExtensionCrawler +cd / +chmod -R go+u-w /opt/ExtensionCrawler +chmod -R go+u-w /usr/local/lib/ +chmod -R go+u-w /usr/local/bin/ +################################################################### + +################################################################### +# Clone cdnjs repository or crate link to external archive dir +ARCHIVE=/shared/brucker_research1/Shared/BrowserExtensions/archive +case ${SINGULARITY_IMAGE} in + *-cdnjs.img) + mkdir -p /opt/archive/filedb + cd /opt/archive/filedb + git clone https://github.com/cdnjs/cdnjs.git cdnjs-git + cd cdnjs-git + git pull + ln -s ${ARCHIVE}/conf . > /dev/null + ln -s ${ARCHIVE}/data > /dev/null + ln -s ${ARCHIVE}/log > /dev/null + ;; + *) + cd /opt/ + ln -s ${ARCHIVE} . + ;; +esac +chmod -R go+u /opt +################################################################### + +################################################################### +# Create mount/bind points for the various network drives +# on SHARC (only useful when using the Singularity image on +# the High-Performance Cluster of The University of Sheffield +mkdir /scratch +mkdir /fastdata +mkdir /data +mkdir /shared + +# Create nvidia driver directories to get rid of the singularity +# warnings on sharc +mkdir /nvbin +mkdir /nvlib +chmod go+u-w /scratch /fastdata /data /shared +################################################################### + +################################################################### +# Manual clean-up and removal of not strictly necessary directories +yes | apt purge g++ +yes | apt autoremove +rm -rf /usr/share/doc || true +################################################################### + +%environment + +export EXTENSION_ARCHIVE=/opt/archive +export PATH=/opt/ExtensionCrawler/:${PATH} + +# We install all python modules into the container, so we do not want +# to use any packages that the user might have installed in their home +# directory. +export PYTHONNOUSERSITE=1 + +%runscript +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# this text will get copied to /singularity and will run whenever the container +# is called as an executable +usage() { + cat < "$LOG" 2>&1 + sudo singularity image.expand --size ${BASESIZE} --writable ${IMAGE} ${BASE}.def > "$LOG" 2>&1 +else + echo "Creating read-only $IMAGE using ${BASE}.def" + sudo singularity build ${IMAGE} ${BASE}.def > "$LOG" 2>&1 +fi + +if [ ! -f $IMAGE ]; then + echo "Image (${IMAGE}) creation failed!" + exit 1 +else + echo "Image (${IMAGE}) creation successful!" +fi + +if [ "$INSTALL" = "true" ]; then + if [ -f $BINDIR/$IMAGE ]; then + mv $BINDIR/$IMAGE $BINDIR/$IMAGE.bak + fi + echo "Installing ${IMAGE} into $BINDIR" + mv $IMAGE $BINDIR +fi