ExtensionCrawler/singularity/ExtensionCrawler.def

#!/bin/sh
# Copyright 2017 The University of Sheffield, UK
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


bootstrap:debootstrap
OSVersion: testing
MirrorURL: https://deb.debian.org/debian 
 
%labels
Maintainer The LogicalHacking Team (https://logicalhacking.com)

%setup 

%post

###################################################################
# Add Debian unstable as a secondary (lower priority) source
# and update the data base of available packages.
cat >> /etc/apt/sources.list << EOF
deb http://ftp.us.debian.org/debian unstable main
EOF

cat > /etc/apt/preferences << EOF
Package: *
Pin: release a=testing
Pin-Priority: 900

Package: *
Pin: release a=unstable
Pin-Priority: 800
EOF

cat > /etc/apt/apt.conf.d/01norecommend << EOF
APT::Install-Recommends "0";
APT::Install-Suggests "0";
EOF

chmod go+r /etc/apt/preferences
apt-get update
###################################################################

###################################################################
# Add hook for apt that removes various files after installation
# that are not needed at runtime.
cat > /etc/apt/apt.conf.d/99-clean << EOF
DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";
EOF
###################################################################

###################################################################
# Configure locales
apt-get install -y locales
echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
echo "en_GB.UTF-8 UTF-8" >> /etc/locale.gen
locale-gen
echo "LANG=en_US.UTF-8" > /etc/default/locale
###################################################################

###################################################################
# Install the core dependencies (Python 3.6 or later)
# from the Debian Testing repository
apt-get	install -y --no-install-recommends libpython3.7-dev python3-magic python3-minimal python3-pip python3-setuptools python3-mysqldb g++ git
apt-get clean
rm -rf /var/lib/apt/lists/* 
###################################################################

###################################################################
# Create /opt for local software (mainly cloned git repositories 
# from logicalhacking.com 
mkdir -p /opt
chmod 755 /opt
###################################################################

###################################################################
# Add the Extension Crawler repository, for more details, visit 
# https://git.logicalhacking.com/BrowserSecurity/ExtensionCrawler
cd /opt 
git clone https://git.logicalhacking.com/BrowserSecurity/ExtensionCrawler.git
cd ExtensionCrawler
git checkout production 
cd ..
pip3 install wheel # simhash needs wheel to build properly, still works without it though
pip3 install --system -e ExtensionCrawler
cd /
chmod -R go+u-w /opt/ExtensionCrawler
chmod -R go+u-w /usr/local/lib/
chmod -R go+u-w /usr/local/bin/
###################################################################

###################################################################
# Clone cdnjs repository or crate link to external archive dir
ARCHIVE=/shared/brucker_research1/Shared/BrowserExtensions/archive
case ${SINGULARITY_IMAGE} in 
  *-cdnjs.img)
    mkdir -p /opt/archive/filedb
    cd /opt/archive/filedb
    git clone https://github.com/cdnjs/cdnjs.git cdnjs-git
    cd cdnjs-git
    git pull
    ln -s ${ARCHIVE}/conf . > /dev/null
    ln -s ${ARCHIVE}/data > /dev/null
    ln -s ${ARCHIVE}/log > /dev/null
    ;;
  *)
    cd  /opt/
    ln -s ${ARCHIVE} .
    ;;
esac
chmod -R go+u /opt
###################################################################

###################################################################
# Create mount/bind points for the various network drives 
# on SHARC (only useful when using the Singularity image on 
# the High-Performance Cluster of The University of Sheffield
mkdir /scratch
mkdir /fastdata
mkdir /data
mkdir /shared

# Create nvidia driver directories to get rid of the singularity
# warnings on sharc
mkdir /nvbin
mkdir /nvlib
chmod go+u-w /scratch /fastdata /data /shared
###################################################################

###################################################################
# Manual clean-up and removal of not strictly necessary directories
yes | apt purge g++
yes | apt autoremove
rm -rf /usr/share/doc || true
###################################################################

%environment

export EXTENSION_ARCHIVE=/opt/archive
export PATH=/opt/ExtensionCrawler/:${PATH}

# We install all python modules into the container, so we do not want
# to use any packages that the user might have installed in their home
# directory.
export PYTHONNOUSERSITE=1

%runscript
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# this text will get copied to /singularity and will run whenever the container
# is called as an executable
usage() {
    cat <<EOF
NAME
     ExtensionCrawler 
SYNOPSIS
    ExtensionCrawler tool [tool options]
    ExtensionCrawler list
    ExtensionCrawler help
DESCRIPTION
    A collection of utilities for downloading and analyzing browser extension 
    from the Chrome Web store.
ENVIRIONMENT
    EXTENSION_ARCHIVE=${EXTENSION_ARCHIVE}
EOF
}

tools() {
    find /opt/ExtensionCrawler  -maxdepth 1 -executable -type f -exec sh -c "{} -h | head -n 1" \;
}

arg="${1:-none}"

case "$arg" in
    none) usage; exit 1;;
    help) usage; exit 0;;
    list) tools; exit 0;;
    # just try to execute it then
    *)    $@;;
esac