Feature/restore backup

.vscode/launch.json

@@ -0,0 +1,35 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "full",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "program": "${workspaceFolder}/scripts/start-backup.js"
+        },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "kickstart",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "program": "${workspaceFolder}/scripts/kickstart.js"
+        },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "deleteAll",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "program": "${workspaceFolder}/scripts/deleteAll.js"
+        }
+    ]
+}

dev-config.json

@@ -0,0 +1,11 @@
+{
+    "params": {
+        "domain": "pryv.me",
+        "domain2": "https://reg.pryv.me/service/info",
+        "backupUsername": "testuserco2",
+        "username": "testimport",
+        "password": "testimport",
+        "includeTrashed": false,
+        "includeAttachments": true
+    }
+}

localhost-config.json

@@ -0,0 +1 @@
+{}

package.json

@@ -21,7 +21,8 @@
     "name": "Pryv"
   },
   "scripts": {
-    "start": "node scripts/start-backup.js",
+    "backup": "node scripts/start-backup.js",
+    "restore": "node scripts/start-restore.js",
     "test": "./node_modules/mocha/bin/mocha test/**/*.test.js",
     "test-travis": "./node_modules/istanbul/lib/cli.js cover ./node_modules/mocha/bin/_mocha -- -R spec ./test/**/*.test.js"
   },
@@ -35,15 +36,22 @@
     "JSONStream": "^1.3.4",
     "async": "~0.9.0",
     "lodash": "^4.16.0",
-    "mkdirp": "~0.5.0",
+    "async-mkdirp": "^1.2.7",
     "pryv": "1.2.1",
-    "read": "^1.0.7"
+    "read": "^1.0.7",
+    "superagent": "^3.5.0",
+    "nconf": "^0.6.x",
+    "winston": "^0.6.x",
+    "parse-domain": "^2.3.4",
+    "bluebird": "^3.7.1",
+    "slug": "^1.1.0"
   },
   "devDependencies": {
     "coveralls": "^3.0.3",
     "istanbul": "^0.4.5",
     "mocha": "^6.1.4",
-    "should": "^11.1.0"
+    "should": "^11.1.0",
+    "json-stringify-safe": "^5.0.1"
   },
   "license": "BSD-3-Clause"
 }

scripts/deleteAll.js

@@ -0,0 +1,84 @@
+/*
+ * This script serves for debugging purpose.
+ * It will delete all Streams and Event from a user.
+ */
+
+const backup = require('../src/main');
+const config = require('../src/utils/config.js');
+const params = config.get('params');
+const { URL } = require('url');
+const BackupDirectory = require('../src/methods/backup-directory');
+const parseDomain = require("parse-domain");
+const Promise = require("bluebird");
+const superagent = require('superagent');
+
+let domain;
+try {
+    new URL(params.domain); // Check if params.domain is a valid url
+
+    const parsedDomain = parseDomain(params.domain); // it is --> we can extract the domain from it
+    domain = parsedDomain.domain + '.' + parsedDomain.tld;
+}
+catch(error) {
+    if(error.code !== 'ERR_INVALID_URL') {
+        console.error(error);
+        return;
+    }
+    domain = params.domain; // it is not, use it as a domain
+}
+
+async function deleteAll() {
+    let conn;
+    try {
+        conn = await Promise.fromCallback(function(callback) {
+            return backup.signInToPryv(params, callback);
+        });
+    } catch (error) {
+        console.error(error);
+        return;
+    }
+
+    const token = conn.auth;
+    const baseUrl = 'https://' + params.username + '.' + params.domain + '/';
+    // await deleteResources('events', baseUrl, token);
+    await deleteResources('streams', baseUrl, token);
+
+    console.log('Delete complete');    
+}
+
+async function deleteResources(resource, baseUrl, token) {
+    let items;
+    try {
+        const res = await superagent.get(baseUrl + resource + '?state=all')
+            .set('Authorization', token)
+            .set('Content-Type', 'application/json');
+        items = res.body[resource];
+    } catch (error) {
+        console.error(error);
+        return;
+    }
+
+    console.log('Deleting ' + items.length + ' ' +resource);
+
+    await asyncForEach(items, async (item) => {
+        console.log('\tdeleting ' + item.id);
+        try {
+            for(let i = 0; i < 2; i++) { // First delete only trash the item
+                await superagent.delete(baseUrl + resource + '/' + item.id + '?mergeEventsWithParent=false')
+                    .set('Authorization', token)
+                    .set('Content-Type', 'application/json');
+                console.log('\t\tpass ' + (i+1) + ' ok');
+            }
+        } catch (error) {
+            console.error(error);
+        }
+    });
+}
+
+async function asyncForEach(array, callback) {
+    for (let index = 0; index < array.length; index++) {
+        await callback(array[index], index, array);
+    }
+}
+
+deleteAll();

scripts/kickstart.js

@@ -0,0 +1,53 @@
+/*
+ * This script serves for debugging purpose.
+ * It will launch a backup task without asking the user all the requested information
+ * such as domain, username, password, etc.
+ * Instead this values are fetched from dev-config.json
+ */
+
+const backup = require('../src/main');
+const config = require('../src/utils/config.js');
+const params = config.get('params');
+const { URL } = require('url');
+const BackupDirectory = require('../src/methods/backup-directory');
+const parseDomain = require("parse-domain");
+
+let domain;
+try {
+    new URL(params.domain); // Check if params.domain is a valid url
+
+    const parsedDomain = parseDomain(params.domain); // it is --> we can extract the domain from it
+    domain = parsedDomain.domain + '.' + parsedDomain.tld;
+}
+catch(error) {
+    if(error.code !== 'ERR_INVALID_URL') {
+        console.error(error);
+        return;
+    }
+    domain = params.domain; // it is not, use it as a domain
+}
+
+async function startAll() {
+    // await startBackup();
+    await startRestore();
+}
+
+async function startBackup() {
+    params.backupDirectory = new BackupDirectory(params.username, domain);
+    await backup.startBackup(params, done.bind(this, 'Backup'));
+}
+
+async function startRestore() {
+    params.backupFolder = new BackupDirectory(params.backupUsername, domain);
+    await backup.startRestore(params, done.bind(this, 'Restore'));
+}
+
+function done(str, error) {
+    if(error) {
+        console.error('Error during ' + str, error);
+        return;
+    }
+    console.log(str + ' completed');
+}
+
+startAll();

scripts/start-backup.js

@@ -1,10 +1,10 @@
-var fs = require('fs'),
-  async = require('async'),
-  read = require('read'),
-  backup = require('../src/main'),
-  BackupDirectory = require('../src/methods/backup-directory');
+const fs = require('fs');
+const async = require('async');
+const read = require('read');
+const backup = require('../src/main');
+const BackupDirectory = require('../src/methods/backup-directory');
 
-var authSettings = {};
+const authSettings = {};
 
 async.series([
   function inputDomain(done) {
@@ -58,7 +58,7 @@ async.series([
     }
   },
   function doBackup(stepDone) {
-    backup.start(authSettings, stepDone);
+    backup.startBackup(authSettings, stepDone);
   }
 ], function (err) {
   if (err) {

scripts/start-restore.js

@@ -0,0 +1,41 @@
+const async = require('async');
+const read = require('read');
+const backup = require('../src/main');
+const BackupDirectory = require('../src/methods/backup-directory');
+
+const authSettings = {};
+
+async.series([
+  function inputDomain(done) {
+    read({prompt: 'Domain (default: pryv.me): ', silent: false}, function (err, domain) {
+      authSettings.domain = domain || 'pryv.me';
+      done(err);
+    });
+  },
+  function inputUsername(done) {
+    read({prompt: 'Username : ', silent: false}, function (err, username) {
+      authSettings.username = username;
+      done(err);
+    });
+  },
+  function inputPassword(done) {
+    read({prompt: 'Password : ', silent: true}, function (err, password) {
+      authSettings.password = password;
+      done(err);
+    });
+  },
+  function inputBackupUsername(done) {
+    read({prompt: 'Backup username : ', silent: false}, function (err, username) {
+      authSettings.backupUsername = username;
+      done(err);
+    });
+  },
+  function doBackup(stepDone) {
+    authSettings.backupFolder = new BackupDirectory(authSettings.backupUsername, authSettings.domain);
+    backup.startRestore(authSettings, stepDone);
+  }
+], function (err) {
+  if (err) {
+    console.log('Failed in process with error', err);
+  }
+});