Moved singularity build scripts into scripts folder.
This commit is contained in:
parent
c835691f23
commit
fdabbcc8c5
|
@ -1,192 +0,0 @@
|
|||
#!/bin/sh
|
||||
# Copyright 2017 The University of Sheffield, UK
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
bootstrap:debootstrap
|
||||
OSVersion: testing
|
||||
MirrorURL: https://deb.debian.org/debian
|
||||
|
||||
%labels
|
||||
Maintainer The LogicalHacking Team (https://logicalhacking.com)
|
||||
|
||||
%setup
|
||||
|
||||
%post
|
||||
|
||||
###################################################################
|
||||
# Add Debian unstable as a secondary (lower priority) source
|
||||
# and update the data base of available packages.
|
||||
cat >> /etc/apt/sources.list << EOF
|
||||
deb http://ftp.us.debian.org/debian unstable main
|
||||
EOF
|
||||
|
||||
cat > /etc/apt/preferences << EOF
|
||||
Package: *
|
||||
Pin: release a=testing
|
||||
Pin-Priority: 900
|
||||
|
||||
Package: *
|
||||
Pin: release a=unstable
|
||||
Pin-Priority: 800
|
||||
EOF
|
||||
|
||||
cat > /etc/apt/apt.conf.d/01norecommend << EOF
|
||||
APT::Install-Recommends "0";
|
||||
APT::Install-Suggests "0";
|
||||
EOF
|
||||
|
||||
chmod go+r /etc/apt/preferences
|
||||
apt-get update
|
||||
###################################################################
|
||||
|
||||
###################################################################
|
||||
# Add hook for apt that removes various files after installation
|
||||
# that are not needed at runtime.
|
||||
cat > /etc/apt/apt.conf.d/99-clean << EOF
|
||||
DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
|
||||
APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
|
||||
Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";
|
||||
EOF
|
||||
###################################################################
|
||||
|
||||
###################################################################
|
||||
# Configure locales
|
||||
apt-get install -y locales
|
||||
echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
|
||||
echo "en_GB.UTF-8 UTF-8" >> /etc/locale.gen
|
||||
locale-gen
|
||||
echo "LANG=en_US.UTF-8" > /etc/default/locale
|
||||
###################################################################
|
||||
|
||||
###################################################################
|
||||
# Install the core dependencies (Python 3.6 or later)
|
||||
# from the Debian Testing repository
|
||||
apt-get install -y --no-install-recommends libpython3.7-dev python3-magic python3-minimal python3-pip python3-setuptools python3-mysqldb g++ git
|
||||
apt-get clean
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
###################################################################
|
||||
|
||||
###################################################################
|
||||
# Create /opt for local software (mainly cloned git repositories
|
||||
# from logicalhacking.com
|
||||
mkdir -p /opt
|
||||
chmod 755 /opt
|
||||
###################################################################
|
||||
|
||||
###################################################################
|
||||
# Add the Extension Crawler repository, for more details, visit
|
||||
# https://git.logicalhacking.com/BrowserSecurity/ExtensionCrawler
|
||||
cd /opt
|
||||
git clone https://git.logicalhacking.com/BrowserSecurity/ExtensionCrawler.git
|
||||
cd ExtensionCrawler
|
||||
git checkout production
|
||||
cd ..
|
||||
pip3 install wheel # simhash needs wheel to build properly, still works without it though
|
||||
pip3 install --system -e ExtensionCrawler
|
||||
cd /
|
||||
chmod -R go+u-w /opt/ExtensionCrawler
|
||||
chmod -R go+u-w /usr/local/lib/
|
||||
chmod -R go+u-w /usr/local/bin/
|
||||
###################################################################
|
||||
|
||||
###################################################################
|
||||
# Clone cdnjs repository or crate link to external archive dir
|
||||
ARCHIVE=/shared/brucker_research1/Shared/BrowserExtensions/archive
|
||||
case ${SINGULARITY_IMAGE} in
|
||||
*-cdnjs.img)
|
||||
mkdir -p /opt/archive/filedb
|
||||
cd /opt/archive/filedb
|
||||
git clone https://github.com/cdnjs/cdnjs.git cdnjs-git
|
||||
cd cdnjs-git
|
||||
git pull
|
||||
ln -s ${ARCHIVE}/conf . > /dev/null
|
||||
ln -s ${ARCHIVE}/data > /dev/null
|
||||
ln -s ${ARCHIVE}/log > /dev/null
|
||||
;;
|
||||
*)
|
||||
cd /opt/
|
||||
ln -s ${ARCHIVE} .
|
||||
;;
|
||||
esac
|
||||
chmod -R go+u /opt
|
||||
###################################################################
|
||||
|
||||
###################################################################
|
||||
# Create mount/bind points for the various network drives
|
||||
# on SHARC (only useful when using the Singularity image on
|
||||
# the High-Performance Cluster of The University of Sheffield
|
||||
mkdir /scratch
|
||||
mkdir /fastdata
|
||||
mkdir /data
|
||||
mkdir /shared
|
||||
|
||||
# Create nvidia driver directories to get rid of the singularity
|
||||
# warnings on sharc
|
||||
mkdir /nvbin
|
||||
mkdir /nvlib
|
||||
chmod go+u-w /scratch /fastdata /data /shared
|
||||
###################################################################
|
||||
|
||||
###################################################################
|
||||
# Manual clean-up and removal of not strictly necessary directories
|
||||
yes | apt purge g++
|
||||
yes | apt autoremove
|
||||
rm -rf /usr/share/doc || true
|
||||
###################################################################
|
||||
|
||||
%environment
|
||||
|
||||
export EXTENSION_ARCHIVE=/opt/archive
|
||||
export PATH=/opt/ExtensionCrawler/:${PATH}
|
||||
|
||||
# We install all python modules into the container, so we do not want
|
||||
# to use any packages that the user might have installed in their home
|
||||
# directory.
|
||||
export PYTHONNOUSERSITE=1
|
||||
|
||||
%runscript
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
# this text will get copied to /singularity and will run whenever the container
|
||||
# is called as an executable
|
||||
usage() {
|
||||
cat <<EOF
|
||||
NAME
|
||||
ExtensionCrawler
|
||||
SYNOPSIS
|
||||
ExtensionCrawler tool [tool options]
|
||||
ExtensionCrawler list
|
||||
ExtensionCrawler help
|
||||
DESCRIPTION
|
||||
A collection of utilities for downloading and analyzing browser extension
|
||||
from the Chrome Web store.
|
||||
ENVIRIONMENT
|
||||
EXTENSION_ARCHIVE=${EXTENSION_ARCHIVE}
|
||||
EOF
|
||||
}
|
||||
|
||||
tools() {
|
||||
find /opt/ExtensionCrawler -maxdepth 1 -executable -type f -exec sh -c "{} -h | head -n 1" \;
|
||||
}
|
||||
|
||||
arg="${1:-none}"
|
||||
|
||||
case "$arg" in
|
||||
none) usage; exit 1;;
|
||||
help) usage; exit 0;;
|
||||
list) tools; exit 0;;
|
||||
# just try to execute it then
|
||||
*) $@;;
|
||||
esac
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
#!/bin/sh
|
||||
# Copyright 2017,2018 The University of Sheffield, UK
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
ARCHIVE="/srv/Shared/BrowserExtensions/archive"
|
||||
BASE=ExtensionCrawler
|
||||
BASESIZE=600
|
||||
|
||||
print_help()
|
||||
{
|
||||
BINDIR=$(dirname "$ARCHIVE")/bin
|
||||
echo "Usage: $prog [OPTION] "
|
||||
echo ""
|
||||
echo "Build a singularity image (fat application) for all ExtensenCrawler utilities."
|
||||
echo ""
|
||||
echo " --help, -h display this help message"
|
||||
echo " --force, -f overwrite existing singularity image"
|
||||
echo " --cdnjs, -c include cdnjs repository (ca. 125 GB)"
|
||||
echo " --install, -i install image (default: $BINDIR)"
|
||||
echo " --archive DIR, -a DIR install image into DIR (default: $ARCHIVE)"
|
||||
}
|
||||
|
||||
|
||||
FORCE="false"
|
||||
CDNJS="false"
|
||||
INSTALL="false"
|
||||
|
||||
while [ $# -gt 0 ]
|
||||
do
|
||||
case "$1" in
|
||||
--force|-f)
|
||||
FORCE="true";;
|
||||
--cdnjs|-c)
|
||||
CDNJS="true";;
|
||||
--install|-i)
|
||||
INSTALL="true";;
|
||||
--archive|-a)
|
||||
ARCHIVE="$2"
|
||||
shift;;
|
||||
--help|-h)
|
||||
print_help
|
||||
exit 0;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
|
||||
if [ "$CDNJS" = "true" ]; then
|
||||
IMAGE=${BASE}-cdnjs.img
|
||||
BASESIZE=$((BASESIZE+134400))
|
||||
else
|
||||
IMAGE=${BASE}.img
|
||||
fi
|
||||
|
||||
BINDIR=$(dirname "$ARCHIVE")/bin
|
||||
|
||||
LOGDIR=$ARCHIVE/log/`date --utc +"%Y-%m"`
|
||||
mkdir -p $LOGDIR
|
||||
LOGPREFIX=$LOGDIR/`date --utc --iso-8601=ns | sed -e 's/:/_/g'`
|
||||
LOG="$LOGPREFIX-$IMAGE.log"
|
||||
|
||||
if [ -f ${IMAGE} ]; then
|
||||
if [ "$FORCE" = "true" ]; then
|
||||
rm -f ${IMAGE}
|
||||
else
|
||||
echo "Image ${IMAGE} exists already."
|
||||
echo "Please remove/rename the image and restart this script"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$CDNJS" = "true" ]; then
|
||||
echo "Creating writable $IMAGE ($BASESIZE MiB) using ${BASE}.def"
|
||||
# TODO: --writable for 'build' action is deprecated due to some sparse file
|
||||
# issues; it is recommended to use --sandbox; however, that creates a
|
||||
# folder, which is probable not what we want here...
|
||||
sudo singularity build --writable ${IMAGE} ${BASE}.def > "$LOG" 2>&1
|
||||
sudo singularity image.expand --size ${BASESIZE} --writable ${IMAGE} ${BASE}.def > "$LOG" 2>&1
|
||||
else
|
||||
echo "Creating read-only $IMAGE using ${BASE}.def"
|
||||
sudo singularity build ${IMAGE} ${BASE}.def > "$LOG" 2>&1
|
||||
fi
|
||||
|
||||
if [ ! -f $IMAGE ]; then
|
||||
echo "Image (${IMAGE}) creation failed!"
|
||||
exit 1
|
||||
else
|
||||
echo "Image (${IMAGE}) creation successful!"
|
||||
fi
|
||||
|
||||
if [ "$INSTALL" = "true" ]; then
|
||||
if [ -f $BINDIR/$IMAGE ]; then
|
||||
mv $BINDIR/$IMAGE $BINDIR/$IMAGE.bak
|
||||
fi
|
||||
echo "Installing ${IMAGE} into $BINDIR"
|
||||
mv $IMAGE $BINDIR
|
||||
fi
|
Loading…
Reference in New Issue