From 3b277f4397b416441f299f9b0d1c1a1c1ea78d0c Mon Sep 17 00:00:00 2001 From: "Achim D. Brucker" Date: Sun, 3 Sep 2017 01:23:12 +0100 Subject: [PATCH] Switched to MariaDB/MySql setup (from SQLite). --- README.md | 10 +- configure.sh | 19 --- paket.dependencies | 2 +- .../LogicalHacking.ExtensionDsLab.fsproj | 134 +----------------- .../SqlConnector.fs | 52 +++---- src/LogicalHacking.ExtensionDsLab/app.config | 17 +++ .../app.config.in | 17 +++ .../paket.references | 1 - src/resources/extensions-schema.sql | 37 ----- src/scripts/example.fsx | 14 +- .../App.config | 10 -- ...LogicalHacking.ExtensionDsLab.Tests.fsproj | 128 +---------------- .../app.config | 0 .../paket.references | 1 - 14 files changed, 67 insertions(+), 375 deletions(-) create mode 100644 src/LogicalHacking.ExtensionDsLab/app.config create mode 100644 src/LogicalHacking.ExtensionDsLab/app.config.in delete mode 100644 src/resources/extensions-schema.sql delete mode 100644 tests/LogicalHacking.ExtensionDsLab.Tests/App.config rename src/LogicalHacking.ExtensionDsLab/App.config => tests/LogicalHacking.ExtensionDsLab.Tests/app.config (100%) diff --git a/README.md b/README.md index 26186d3..cc33004 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,7 @@ security and privacy related properties of the browser extensions (and Chrome Apps) offered in the [Chrome Web Store](https://chrome.google.com/webstore/). # Dependencies -Dependencies are managed by paket, with the exception of -* the SQLite libraries required by the SQLProvider. For Mono, - ``configure.sh`` tries to set up everything nicely using a - system-wide installed ``Mono.Data.Sqlite.dll``. If this fails, - please follow the steps documented at the - [SQLProvider website](http://fsprojects.github.io/SQLProvider/core/sqlite.html). -* the ``sqlite3`` binary is used for creating an empty data base used - by the SQL type provider +Dependencies are managed by paket. # Installation * On Linux/MacOs using mono: @@ -26,7 +19,6 @@ Dependencies are managed by paket, with the exception of * On Windows (untested), you need to set up the initial project manually: ``` - type src/resources/scholar-kpi-schema.sql | sqlite3 src/resources/scholar-kpi-schema.sqlite .paket/paket.bootstrapper.exe .paket/paket.exe install ```` diff --git a/configure.sh b/configure.sh index e0acf3c..38eae6c 100755 --- a/configure.sh +++ b/configure.sh @@ -21,22 +21,3 @@ exit_code=$? if [ $exit_code -ne 0 ]; then exit $exit_code fi -if test "$OS" = "Windows_NT" -then - echo "Please install System.Data.SQLite.dll into packages/SQLProvider/lib." -else - if [ -f /usr/lib/mono/4.5/Mono.Data.Sqlite.dll ]; - then - if [ ! -f packages/SQLProvider/lib/Mono.Data.Sqlite.dll ]; - then - ln -s /usr/lib/mono/4.5/Mono.Data.Sqlite.dll packages/SQLProvider/lib - fi - else - echo "Please install Mono.Data.Sqlite.dll into packages/SQLProvider/lib." - fi -fi - -echo "Creating empty database for type provider." -rm -f src/resources/extensions-schema.sqlite -sqlite3 src/resources/extensions-schema.sqlite < src/resources/extensions-schema.sql - diff --git a/paket.dependencies b/paket.dependencies index 3940ba9..38b57c6 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -10,8 +10,8 @@ source https://nuget.org/api/v2 nuget FSharp.Core ~> 4.0.0.1 redirects: force nuget FsLab +nuget Mysql.Data nuget SQLProvider -nuget System.Data.Sqlite group Build source https://nuget.org/api/v2 diff --git a/src/LogicalHacking.ExtensionDsLab/LogicalHacking.ExtensionDsLab.fsproj b/src/LogicalHacking.ExtensionDsLab/LogicalHacking.ExtensionDsLab.fsproj index f5703bb..fb51821 100644 --- a/src/LogicalHacking.ExtensionDsLab/LogicalHacking.ExtensionDsLab.fsproj +++ b/src/LogicalHacking.ExtensionDsLab/LogicalHacking.ExtensionDsLab.fsproj @@ -50,23 +50,6 @@ - - - - <__paket__System_Data_SQLite_Core_targets>net45\System.Data.SQLite.Core - - - - - <__paket__System_Data_SQLite_Core_targets>net451\System.Data.SQLite.Core - - - - - <__paket__System_Data_SQLite_Core_targets>net46\System.Data.SQLite.Core - - - @@ -75,14 +58,13 @@ True - + / - - + @@ -116,25 +98,6 @@ - - - - - True - - - ..\..\packages\EntityFramework\lib\net45\EntityFramework.dll - True - True - - - ..\..\packages\EntityFramework\lib\net45\EntityFramework.SqlServer.dll - True - True - - - - @@ -266,6 +229,11 @@ True True + + ..\..\packages\MySql.Data\lib\net45\MySql.Data.dll + True + True + @@ -280,93 +248,6 @@ - - - - - ..\..\packages\System.Data.SQLite.Core\lib\net45\System.Data.SQLite.dll - True - True - - - - - - - ..\..\packages\System.Data.SQLite.Core\lib\net451\System.Data.SQLite.dll - True - True - - - - - - - ..\..\packages\System.Data.SQLite.Core\lib\net46\System.Data.SQLite.dll - True - True - - - - - - - - - ..\..\packages\System.Data.SQLite.EF6\lib\net45\System.Data.SQLite.EF6.dll - True - True - - - - - - - ..\..\packages\System.Data.SQLite.EF6\lib\net451\System.Data.SQLite.EF6.dll - True - True - - - - - - - ..\..\packages\System.Data.SQLite.EF6\lib\net46\System.Data.SQLite.EF6.dll - True - True - - - - - - - - - ..\..\packages\System.Data.SQLite.Linq\lib\net45\System.Data.SQLite.Linq.dll - True - True - - - - - - - ..\..\packages\System.Data.SQLite.Linq\lib\net451\System.Data.SQLite.Linq.dll - True - True - - - - - - - ..\..\packages\System.Data.SQLite.Linq\lib\net46\System.Data.SQLite.Linq.dll - True - True - - - - @@ -400,5 +281,4 @@ - diff --git a/src/LogicalHacking.ExtensionDsLab/SqlConnector.fs b/src/LogicalHacking.ExtensionDsLab/SqlConnector.fs index 2f44888..b10a58c 100644 --- a/src/LogicalHacking.ExtensionDsLab/SqlConnector.fs +++ b/src/LogicalHacking.ExtensionDsLab/SqlConnector.fs @@ -21,55 +21,35 @@ open FSharp.Data.Sql open System.IO module SqlConnector = - let [] DevelopmentDB = "aa-ac.sqlite" - let [] FullDB = "full.sqlite" - let [] DatabaseDir = "archive/db" - // static (compile time) ConnectionString - let [] ConnectionString = @"Data Source=" - + __SOURCE_DIRECTORY__ - + @"/../resources/extensions-schema.sqlite" - + @";Version=3" + [] + let ConnectionStringName = @"ProductionExtensionDB" + [] + let ResolutionPath = __SOURCE_DIRECTORY__ + @"/../../packages/MySql.Data/lib/net45" + // create a type alias with the connection string and database vendor settings type ExtensionDbProvider = SqlDataProvider< - ConnectionString = ConnectionString, - DatabaseVendor = Common.DatabaseProviderTypes.SQLITE, - // ResolutionPath = ResolutionPath, + DatabaseVendor = Common.DatabaseProviderTypes.MYSQL, + ConnectionStringName = ConnectionStringName, + ResolutionPath = ResolutionPath, IndividualsAmount = 500, UseOptionTypes = true > type ExtensionDbType = ExtensionDbProvider.dataContext - type Db = Dev | Prod | Auto | Custom of string - let rec getCtx dir = let archiveDir = function - | None -> __SOURCE_DIRECTORY__ + @"/../.." - | Some s -> s - function - | Dev -> ExtensionDbProvider.GetDataContext(sprintf "Data Source=%s/%s/%s;Version=3" - (archiveDir dir) DatabaseDir DevelopmentDB) - | Prod -> ExtensionDbProvider.GetDataContext(sprintf "Data Source=%s/%s/%s;Version=3" - (archiveDir dir) DatabaseDir FullDB) - | Custom db -> ExtensionDbProvider.GetDataContext(sprintf "Data Source=%s/%s/%s;Version=3" - (archiveDir dir) DatabaseDir db) - | Auto -> if File.Exists(sprintf "%s/%s/%s" (archiveDir dir) DatabaseDir FullDB) then - getCtx dir Prod - else - getCtx dir Dev - let getDownloads (ctx:ExtensionDbType) extid = query { - for order in ctx.Main.Extension do - where (order.Extid = Some extid) + for order in ctx.Extensions.Extension do + where (order.Extid = extid) select (order.Date, order.Downloads) - } |> Seq.filter (fun (date, download) -> date.IsSome && download.IsSome) - |> Seq.map (fun (date, download) -> (System.DateTime.Parse(date.Value), + } |> Seq.filter (fun (_, download) -> download.IsSome) + |> Seq.map (fun (date, download) -> (date, download.Value)) |> Seq.toArray let getAllExtIds (ctx:ExtensionDbType) = query { - for ext in ctx.Main.Extension do - select ext.Extid - } |> Seq.distinct - |> Seq.choose id - |> Seq.toList + for ext in ctx.Extensions.Extension do + select ext.Extid + distinct + } |> Seq.toList \ No newline at end of file diff --git a/src/LogicalHacking.ExtensionDsLab/app.config b/src/LogicalHacking.ExtensionDsLab/app.config new file mode 100644 index 0000000..feb7435 --- /dev/null +++ b/src/LogicalHacking.ExtensionDsLab/app.config @@ -0,0 +1,17 @@ + + + + + + + + + True + + + + + + diff --git a/src/LogicalHacking.ExtensionDsLab/app.config.in b/src/LogicalHacking.ExtensionDsLab/app.config.in new file mode 100644 index 0000000..4821e49 --- /dev/null +++ b/src/LogicalHacking.ExtensionDsLab/app.config.in @@ -0,0 +1,17 @@ + + + + + + + + + True + + + + + + diff --git a/src/LogicalHacking.ExtensionDsLab/paket.references b/src/LogicalHacking.ExtensionDsLab/paket.references index 5fde82f..fca5a99 100644 --- a/src/LogicalHacking.ExtensionDsLab/paket.references +++ b/src/LogicalHacking.ExtensionDsLab/paket.references @@ -1,4 +1,3 @@ FSharp.Core FsLab -System.Data.Sqlite SQLProvider \ No newline at end of file diff --git a/src/resources/extensions-schema.sql b/src/resources/extensions-schema.sql deleted file mode 100644 index 337df17..0000000 --- a/src/resources/extensions-schema.sql +++ /dev/null @@ -1,37 +0,0 @@ -CREATE TABLE support(author, commentdate, extid, date, displayname, title, language, shortauthor, comment, PRIMARY KEY (author, commentdate, extid, date)); -CREATE VIRTUAL TABLE support_fts using fts4(content="support", author, commentdate, extid, date, displayname, title, language, shortauthor, comment); -CREATE TABLE IF NOT EXISTS 'support_fts_segments'(blockid INTEGER PRIMARY KEY, block BLOB); -CREATE TABLE IF NOT EXISTS 'support_fts_segdir'(level INTEGER,idx INTEGER,start_block INTEGER,leaves_end_block INTEGER,end_block INTEGER,root BLOB,PRIMARY KEY(level, idx)); -CREATE TABLE IF NOT EXISTS 'support_fts_docsize'(docid INTEGER PRIMARY KEY, size BLOB); -CREATE TABLE IF NOT EXISTS 'support_fts_stat'(id INTEGER PRIMARY KEY, value BLOB); -CREATE TRIGGER support_bu BEFORE UPDATE ON support BEGIN DELETE FROM support_fts WHERE docid=old.rowid;END; -CREATE TRIGGER support_bd BEFORE DELETE ON support BEGIN DELETE FROM support_fts WHERE docid=old.rowid;END; -CREATE TRIGGER support_au AFTER UPDATE ON support BEGIN INSERT INTO support_fts(docid, author, commentdate, extid, date, displayname, title, language, shortauthor, comment) VALUES(new.rowid, new.author, new.commentdate, new.extid, new.date, new.displayname, new.title, new.language, new.shortauthor, new.comment);END; -CREATE TRIGGER support_ai AFTER INSERT ON support BEGIN INSERT INTO support_fts(docid, author, commentdate, extid, date, displayname, title, language, shortauthor, comment) VALUES(new.rowid, new.author, new.commentdate, new.extid, new.date, new.displayname, new.title, new.language, new.shortauthor, new.comment);END; -CREATE TABLE review(author, commentdate, extid, date, displayname, rating, language, shortauthor, comment, PRIMARY KEY (author, commentdate, extid, date)); -CREATE VIRTUAL TABLE review_fts using fts4(content="review", author, commentdate, extid, date, displayname, rating, language, shortauthor, comment); -CREATE TABLE IF NOT EXISTS 'review_fts_segments'(blockid INTEGER PRIMARY KEY, block BLOB); -CREATE TABLE IF NOT EXISTS 'review_fts_segdir'(level INTEGER,idx INTEGER,start_block INTEGER,leaves_end_block INTEGER,end_block INTEGER,root BLOB,PRIMARY KEY(level, idx)); -CREATE TABLE IF NOT EXISTS 'review_fts_docsize'(docid INTEGER PRIMARY KEY, size BLOB); -CREATE TABLE IF NOT EXISTS 'review_fts_stat'(id INTEGER PRIMARY KEY, value BLOB); -CREATE TRIGGER review_bu BEFORE UPDATE ON review BEGIN DELETE FROM review_fts WHERE docid=old.rowid;END; -CREATE TRIGGER review_bd BEFORE DELETE ON review BEGIN DELETE FROM review_fts WHERE docid=old.rowid;END; -CREATE TRIGGER review_au AFTER UPDATE ON review BEGIN INSERT INTO review_fts(docid, author, commentdate, extid, date, displayname, rating, language, shortauthor, comment) VALUES(new.rowid, new.author, new.commentdate, new.extid, new.date, new.displayname, new.rating, new.language, new.shortauthor, new.comment);END; -CREATE TRIGGER review_ai AFTER INSERT ON review BEGIN INSERT INTO review_fts(docid, author, commentdate, extid, date, displayname, rating, language, shortauthor, comment) VALUES(new.rowid, new.author, new.commentdate, new.extid, new.date, new.displayname, new.rating, new.language, new.shortauthor, new.comment);END; -CREATE TABLE reply(author, commentdate, extid, date, displayname, replyto, language, shortauthor, comment, PRIMARY KEY (author, commentdate, extid, date)); -CREATE VIRTUAL TABLE reply_fts using fts4(content="reply", author, commentdate, extid, date, displayname, replyto, language, shortauthor, comment); -CREATE TABLE IF NOT EXISTS 'reply_fts_segments'(blockid INTEGER PRIMARY KEY, block BLOB); -CREATE TABLE IF NOT EXISTS 'reply_fts_segdir'(level INTEGER,idx INTEGER,start_block INTEGER,leaves_end_block INTEGER,end_block INTEGER,root BLOB,PRIMARY KEY(level, idx)); -CREATE TABLE IF NOT EXISTS 'reply_fts_docsize'(docid INTEGER PRIMARY KEY, size BLOB); -CREATE TABLE IF NOT EXISTS 'reply_fts_stat'(id INTEGER PRIMARY KEY, value BLOB); -CREATE TRIGGER reply_bu BEFORE UPDATE ON reply BEGIN DELETE FROM reply_fts WHERE docid=old.rowid;END; -CREATE TRIGGER reply_bd BEFORE DELETE ON reply BEGIN DELETE FROM reply_fts WHERE docid=old.rowid;END; -CREATE TRIGGER reply_au AFTER UPDATE ON reply BEGIN INSERT INTO reply_fts(docid, author, commentdate, extid, date, displayname, replyto, language, shortauthor, comment) VALUES(new.rowid, new.author, new.commentdate, new.extid, new.date, new.displayname, new.replyto, new.language, new.shortauthor, new.comment);END; -CREATE TRIGGER reply_ai AFTER INSERT ON reply BEGIN INSERT INTO reply_fts(docid, author, commentdate, extid, date, displayname, replyto, language, shortauthor, comment) VALUES(new.rowid, new.author, new.commentdate, new.extid, new.date, new.displayname, new.replyto, new.language, new.shortauthor, new.comment);END; -CREATE TABLE category (extid TEXT,date TEXT,category TEXT,PRIMARY KEY (extid, date, category)); -CREATE TABLE content_script_url (crx_etag TEXT,url TEXT,PRIMARY KEY (crx_etag, url)); -CREATE TABLE permission (crx_etag TEXT,permission TEXT,PRIMARY KEY (crx_etag, permission)); -CREATE TABLE crx (crx_etag TEXT PRIMARY KEY,filename TEXT,size INTEGER,publickey BLOB); -CREATE TABLE jsfile (crx_etag TEXT,path TEXT,size INTEGER,md5 TEXT,PRIMARY KEY (crx_etag, path)); -CREATE TABLE status (extid TEXT,date TEXT,crx_status INTEGER,overview_status INTEGER,overview_exception TEXT,PRIMARY KEY (extid, date)); -CREATE TABLE extension (extid TEXT,date TEXT,name TEXT,version TEXT,description TEXT,downloads INTEGER,rating REAL,ratingcount INTEGER,fulldescription TEXT,developer TEXT,itemcategory TEXT,crx_etag TEXT,lastupdated TEXT,PRIMARY KEY (extid, date),FOREIGN KEY (crx_etag) REFERENCES crx(crx_etag)); diff --git a/src/scripts/example.fsx b/src/scripts/example.fsx index 4aa269e..2472907 100644 --- a/src/scripts/example.fsx +++ b/src/scripts/example.fsx @@ -34,22 +34,20 @@ open XPlot.GoogleCharts.Deedle open LogicalHacking.ExtensionDsLab.Archive.SqlConnector -let ctx = getCtx None Auto +let ctx = ExtensionDbProvider.GetDataContext() let chartDownloads extIds = (List.map (fun e -> (series (getDownloads ctx e))) extIds) |> Chart.Line |> Chart.WithOptions (Options(legend=Legend(position="bottom"))) |> Chart.WithLabels extIds -chartDownloads (List.take 5 (getAllExtIds ctx)) +chartDownloads ["lkllajgbhondgjjnhmmgbjndmogapinp"] +// chartDownloads (List.take 5 (getAllExtIds ctx)) let extData = - query {for x in ctx.Main.Extension do + query {for x in ctx.Extensions.Extension do select (x.Date, x.Name, x.Downloads) } - |> Seq.filter (fun (date, name, download) -> date.IsSome && name.IsSome && download.IsSome) - |> Seq.map (fun (date, name, download) -> (System.DateTime.Parse(date.Value), name.Value, - download.Value)) + |> Seq.filter (fun (date, name, download) -> name.IsSome && download.IsSome) + |> Seq.map (fun (date, name, download) -> (date, name.Value, download.Value)) |> Frame.ofRecords |> Frame.indexColsWith ["Date"; "Name"; "Downloads"] - - diff --git a/tests/LogicalHacking.ExtensionDsLab.Tests/App.config b/tests/LogicalHacking.ExtensionDsLab.Tests/App.config deleted file mode 100644 index 7c7a547..0000000 --- a/tests/LogicalHacking.ExtensionDsLab.Tests/App.config +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - True - - - - diff --git a/tests/LogicalHacking.ExtensionDsLab.Tests/LogicalHacking.ExtensionDsLab.Tests.fsproj b/tests/LogicalHacking.ExtensionDsLab.Tests/LogicalHacking.ExtensionDsLab.Tests.fsproj index 0090446..d6880f9 100644 --- a/tests/LogicalHacking.ExtensionDsLab.Tests/LogicalHacking.ExtensionDsLab.Tests.fsproj +++ b/tests/LogicalHacking.ExtensionDsLab.Tests/LogicalHacking.ExtensionDsLab.Tests.fsproj @@ -53,23 +53,6 @@ - - - - <__paket__System_Data_SQLite_Core_targets>net45\System.Data.SQLite.Core - - - - - <__paket__System_Data_SQLite_Core_targets>net451\System.Data.SQLite.Core - - - - - <__paket__System_Data_SQLite_Core_targets>net46\System.Data.SQLite.Core - - -