Moved singularity build scripts into scripts folder.
This commit is contained in:
parent
c835691f23
commit
fdabbcc8c5
|
@ -1,192 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
# Copyright 2017 The University of Sheffield, UK
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
|
|
||||||
bootstrap:debootstrap
|
|
||||||
OSVersion: testing
|
|
||||||
MirrorURL: https://deb.debian.org/debian
|
|
||||||
|
|
||||||
%labels
|
|
||||||
Maintainer The LogicalHacking Team (https://logicalhacking.com)
|
|
||||||
|
|
||||||
%setup
|
|
||||||
|
|
||||||
%post
|
|
||||||
|
|
||||||
###################################################################
|
|
||||||
# Add Debian unstable as a secondary (lower priority) source
|
|
||||||
# and update the data base of available packages.
|
|
||||||
cat >> /etc/apt/sources.list << EOF
|
|
||||||
deb http://ftp.us.debian.org/debian unstable main
|
|
||||||
EOF
|
|
||||||
|
|
||||||
cat > /etc/apt/preferences << EOF
|
|
||||||
Package: *
|
|
||||||
Pin: release a=testing
|
|
||||||
Pin-Priority: 900
|
|
||||||
|
|
||||||
Package: *
|
|
||||||
Pin: release a=unstable
|
|
||||||
Pin-Priority: 800
|
|
||||||
EOF
|
|
||||||
|
|
||||||
cat > /etc/apt/apt.conf.d/01norecommend << EOF
|
|
||||||
APT::Install-Recommends "0";
|
|
||||||
APT::Install-Suggests "0";
|
|
||||||
EOF
|
|
||||||
|
|
||||||
chmod go+r /etc/apt/preferences
|
|
||||||
apt-get update
|
|
||||||
###################################################################
|
|
||||||
|
|
||||||
###################################################################
|
|
||||||
# Add hook for apt that removes various files after installation
|
|
||||||
# that are not needed at runtime.
|
|
||||||
cat > /etc/apt/apt.conf.d/99-clean << EOF
|
|
||||||
DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
|
|
||||||
APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
|
|
||||||
Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";
|
|
||||||
EOF
|
|
||||||
###################################################################
|
|
||||||
|
|
||||||
###################################################################
|
|
||||||
# Configure locales
|
|
||||||
apt-get install -y locales
|
|
||||||
echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
|
|
||||||
echo "en_GB.UTF-8 UTF-8" >> /etc/locale.gen
|
|
||||||
locale-gen
|
|
||||||
echo "LANG=en_US.UTF-8" > /etc/default/locale
|
|
||||||
###################################################################
|
|
||||||
|
|
||||||
###################################################################
|
|
||||||
# Install the core dependencies (Python 3.6 or later)
|
|
||||||
# from the Debian Testing repository
|
|
||||||
apt-get install -y --no-install-recommends libpython3.7-dev python3-magic python3-minimal python3-pip python3-setuptools python3-mysqldb g++ git
|
|
||||||
apt-get clean
|
|
||||||
rm -rf /var/lib/apt/lists/*
|
|
||||||
###################################################################
|
|
||||||
|
|
||||||
###################################################################
|
|
||||||
# Create /opt for local software (mainly cloned git repositories
|
|
||||||
# from logicalhacking.com
|
|
||||||
mkdir -p /opt
|
|
||||||
chmod 755 /opt
|
|
||||||
###################################################################
|
|
||||||
|
|
||||||
###################################################################
|
|
||||||
# Add the Extension Crawler repository, for more details, visit
|
|
||||||
# https://git.logicalhacking.com/BrowserSecurity/ExtensionCrawler
|
|
||||||
cd /opt
|
|
||||||
git clone https://git.logicalhacking.com/BrowserSecurity/ExtensionCrawler.git
|
|
||||||
cd ExtensionCrawler
|
|
||||||
git checkout production
|
|
||||||
cd ..
|
|
||||||
pip3 install wheel # simhash needs wheel to build properly, still works without it though
|
|
||||||
pip3 install --system -e ExtensionCrawler
|
|
||||||
cd /
|
|
||||||
chmod -R go+u-w /opt/ExtensionCrawler
|
|
||||||
chmod -R go+u-w /usr/local/lib/
|
|
||||||
chmod -R go+u-w /usr/local/bin/
|
|
||||||
###################################################################
|
|
||||||
|
|
||||||
###################################################################
|
|
||||||
# Clone cdnjs repository or crate link to external archive dir
|
|
||||||
ARCHIVE=/shared/brucker_research1/Shared/BrowserExtensions/archive
|
|
||||||
case ${SINGULARITY_IMAGE} in
|
|
||||||
*-cdnjs.img)
|
|
||||||
mkdir -p /opt/archive/filedb
|
|
||||||
cd /opt/archive/filedb
|
|
||||||
git clone https://github.com/cdnjs/cdnjs.git cdnjs-git
|
|
||||||
cd cdnjs-git
|
|
||||||
git pull
|
|
||||||
ln -s ${ARCHIVE}/conf . > /dev/null
|
|
||||||
ln -s ${ARCHIVE}/data > /dev/null
|
|
||||||
ln -s ${ARCHIVE}/log > /dev/null
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
cd /opt/
|
|
||||||
ln -s ${ARCHIVE} .
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
chmod -R go+u /opt
|
|
||||||
###################################################################
|
|
||||||
|
|
||||||
###################################################################
|
|
||||||
# Create mount/bind points for the various network drives
|
|
||||||
# on SHARC (only useful when using the Singularity image on
|
|
||||||
# the High-Performance Cluster of The University of Sheffield
|
|
||||||
mkdir /scratch
|
|
||||||
mkdir /fastdata
|
|
||||||
mkdir /data
|
|
||||||
mkdir /shared
|
|
||||||
|
|
||||||
# Create nvidia driver directories to get rid of the singularity
|
|
||||||
# warnings on sharc
|
|
||||||
mkdir /nvbin
|
|
||||||
mkdir /nvlib
|
|
||||||
chmod go+u-w /scratch /fastdata /data /shared
|
|
||||||
###################################################################
|
|
||||||
|
|
||||||
###################################################################
|
|
||||||
# Manual clean-up and removal of not strictly necessary directories
|
|
||||||
yes | apt purge g++
|
|
||||||
yes | apt autoremove
|
|
||||||
rm -rf /usr/share/doc || true
|
|
||||||
###################################################################
|
|
||||||
|
|
||||||
%environment
|
|
||||||
|
|
||||||
export EXTENSION_ARCHIVE=/opt/archive
|
|
||||||
export PATH=/opt/ExtensionCrawler/:${PATH}
|
|
||||||
|
|
||||||
# We install all python modules into the container, so we do not want
|
|
||||||
# to use any packages that the user might have installed in their home
|
|
||||||
# directory.
|
|
||||||
export PYTHONNOUSERSITE=1
|
|
||||||
|
|
||||||
%runscript
|
|
||||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
# this text will get copied to /singularity and will run whenever the container
|
|
||||||
# is called as an executable
|
|
||||||
usage() {
|
|
||||||
cat <<EOF
|
|
||||||
NAME
|
|
||||||
ExtensionCrawler
|
|
||||||
SYNOPSIS
|
|
||||||
ExtensionCrawler tool [tool options]
|
|
||||||
ExtensionCrawler list
|
|
||||||
ExtensionCrawler help
|
|
||||||
DESCRIPTION
|
|
||||||
A collection of utilities for downloading and analyzing browser extension
|
|
||||||
from the Chrome Web store.
|
|
||||||
ENVIRIONMENT
|
|
||||||
EXTENSION_ARCHIVE=${EXTENSION_ARCHIVE}
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
tools() {
|
|
||||||
find /opt/ExtensionCrawler -maxdepth 1 -executable -type f -exec sh -c "{} -h | head -n 1" \;
|
|
||||||
}
|
|
||||||
|
|
||||||
arg="${1:-none}"
|
|
||||||
|
|
||||||
case "$arg" in
|
|
||||||
none) usage; exit 1;;
|
|
||||||
help) usage; exit 0;;
|
|
||||||
list) tools; exit 0;;
|
|
||||||
# just try to execute it then
|
|
||||||
*) $@;;
|
|
||||||
esac
|
|
||||||
|
|
|
@ -1,108 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
# Copyright 2017,2018 The University of Sheffield, UK
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
ARCHIVE="/srv/Shared/BrowserExtensions/archive"
|
|
||||||
BASE=ExtensionCrawler
|
|
||||||
BASESIZE=600
|
|
||||||
|
|
||||||
print_help()
|
|
||||||
{
|
|
||||||
BINDIR=$(dirname "$ARCHIVE")/bin
|
|
||||||
echo "Usage: $prog [OPTION] "
|
|
||||||
echo ""
|
|
||||||
echo "Build a singularity image (fat application) for all ExtensenCrawler utilities."
|
|
||||||
echo ""
|
|
||||||
echo " --help, -h display this help message"
|
|
||||||
echo " --force, -f overwrite existing singularity image"
|
|
||||||
echo " --cdnjs, -c include cdnjs repository (ca. 125 GB)"
|
|
||||||
echo " --install, -i install image (default: $BINDIR)"
|
|
||||||
echo " --archive DIR, -a DIR install image into DIR (default: $ARCHIVE)"
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
FORCE="false"
|
|
||||||
CDNJS="false"
|
|
||||||
INSTALL="false"
|
|
||||||
|
|
||||||
while [ $# -gt 0 ]
|
|
||||||
do
|
|
||||||
case "$1" in
|
|
||||||
--force|-f)
|
|
||||||
FORCE="true";;
|
|
||||||
--cdnjs|-c)
|
|
||||||
CDNJS="true";;
|
|
||||||
--install|-i)
|
|
||||||
INSTALL="true";;
|
|
||||||
--archive|-a)
|
|
||||||
ARCHIVE="$2"
|
|
||||||
shift;;
|
|
||||||
--help|-h)
|
|
||||||
print_help
|
|
||||||
exit 0;;
|
|
||||||
esac
|
|
||||||
shift
|
|
||||||
done
|
|
||||||
|
|
||||||
|
|
||||||
if [ "$CDNJS" = "true" ]; then
|
|
||||||
IMAGE=${BASE}-cdnjs.img
|
|
||||||
BASESIZE=$((BASESIZE+134400))
|
|
||||||
else
|
|
||||||
IMAGE=${BASE}.img
|
|
||||||
fi
|
|
||||||
|
|
||||||
BINDIR=$(dirname "$ARCHIVE")/bin
|
|
||||||
|
|
||||||
LOGDIR=$ARCHIVE/log/`date --utc +"%Y-%m"`
|
|
||||||
mkdir -p $LOGDIR
|
|
||||||
LOGPREFIX=$LOGDIR/`date --utc --iso-8601=ns | sed -e 's/:/_/g'`
|
|
||||||
LOG="$LOGPREFIX-$IMAGE.log"
|
|
||||||
|
|
||||||
if [ -f ${IMAGE} ]; then
|
|
||||||
if [ "$FORCE" = "true" ]; then
|
|
||||||
rm -f ${IMAGE}
|
|
||||||
else
|
|
||||||
echo "Image ${IMAGE} exists already."
|
|
||||||
echo "Please remove/rename the image and restart this script"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$CDNJS" = "true" ]; then
|
|
||||||
echo "Creating writable $IMAGE ($BASESIZE MiB) using ${BASE}.def"
|
|
||||||
# TODO: --writable for 'build' action is deprecated due to some sparse file
|
|
||||||
# issues; it is recommended to use --sandbox; however, that creates a
|
|
||||||
# folder, which is probable not what we want here...
|
|
||||||
sudo singularity build --writable ${IMAGE} ${BASE}.def > "$LOG" 2>&1
|
|
||||||
sudo singularity image.expand --size ${BASESIZE} --writable ${IMAGE} ${BASE}.def > "$LOG" 2>&1
|
|
||||||
else
|
|
||||||
echo "Creating read-only $IMAGE using ${BASE}.def"
|
|
||||||
sudo singularity build ${IMAGE} ${BASE}.def > "$LOG" 2>&1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ ! -f $IMAGE ]; then
|
|
||||||
echo "Image (${IMAGE}) creation failed!"
|
|
||||||
exit 1
|
|
||||||
else
|
|
||||||
echo "Image (${IMAGE}) creation successful!"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$INSTALL" = "true" ]; then
|
|
||||||
if [ -f $BINDIR/$IMAGE ]; then
|
|
||||||
mv $BINDIR/$IMAGE $BINDIR/$IMAGE.bak
|
|
||||||
fi
|
|
||||||
echo "Installing ${IMAGE} into $BINDIR"
|
|
||||||
mv $IMAGE $BINDIR
|
|
||||||
fi
|
|
Loading…
Reference in New Issue