From 0dfb26f8fa390bd83c5c457fa86da51fd9885d54 Mon Sep 17 00:00:00 2001 From: Daniel Scalzi Date: Sat, 13 May 2017 04:24:15 -0400 Subject: [PATCH] Progress towards final asset downloading system (assetguard). --- app/assets/js/assetdownload.js | 18 +- app/assets/js/assetguard.js | 299 +++++++++++++++++++++++++++++++++ app/assets/js/library.js | 37 ---- app/assets/js/script.js | 15 +- index.js | 13 -- 5 files changed, 320 insertions(+), 62 deletions(-) create mode 100644 app/assets/js/assetguard.js delete mode 100644 app/assets/js/library.js diff --git a/app/assets/js/assetdownload.js b/app/assets/js/assetdownload.js index 7bbd8eab..7a17fe7c 100644 --- a/app/assets/js/assetdownload.js +++ b/app/assets/js/assetdownload.js @@ -4,9 +4,13 @@ const path = require('path') const mkpath = require('mkdirp'); const async = require('async') const crypto = require('crypto') -const library = require('./library.js') +const Library = require('./library.js') const {BrowserWindow} = require('electron') +/** + * PHASING THIS OUT, WILL BE REMOVED WHEN ASSET GUARD MODULE IS COMPLETE! + */ + function Asset(from, to, size, hash){ this.from = from this.to = to @@ -22,14 +26,6 @@ function AssetIndex(id, sha1, size, url, totalSize){ this.totalSize = totalSize } -function Library(id, sha1, size, from, to){ - this.id = id - this.sha1 = sha1 - this.size = size - this.from = from - this.to = to -} - /** * This function will download the version index data and read it into a Javascript * Object. This object will then be returned. @@ -109,12 +105,12 @@ downloadLibraries = function(versionData, basePath){ //Check validity of each library. If the hashs don't match, download the library. libArr.forEach(function(lib, index){ - if(library.validateRules(lib.rules)){ + if(Library.validateRules(lib.rules)){ let artifact = null if(lib.natives == null){ artifact = lib.downloads.artifact } else { - artifact = lib.downloads.classifiers[lib.natives[library.mojangFriendlyOS()]] + artifact = lib.downloads.classifiers[lib.natives[Library.mojangFriendlyOS()]] } const libItm = new Library(lib.name, artifact.sha1, artifact.size, artifact.url, path.join(libPath, artifact.path)) if(!validateLocalIntegrity(libItm.to, 'sha1', libItm.sha1)){ diff --git a/app/assets/js/assetguard.js b/app/assets/js/assetguard.js new file mode 100644 index 00000000..c3a9f31e --- /dev/null +++ b/app/assets/js/assetguard.js @@ -0,0 +1,299 @@ +/* Requirements */ +const fs = require('fs') +const request = require('request') +const path = require('path') +const mkpath = require('mkdirp'); +const async = require('async') +const crypto = require('crypto') +const EventEmitter = require('events'); +const {remote} = require('electron') + +/* Classes */ + +class Asset{ + constructor(id, hash, size, from, to){ + this.id = id + this.hash = hash + this.size = size + this.from = from + this.to = to + } +} + +class Library extends Asset{ + + static mojangFriendlyOS(){ + const opSys = process.platform + if (opSys === 'darwin') { + return 'osx'; + } else if (opSys === 'win32'){ + return 'windows'; + } else if (opSys === 'linux'){ + return 'linux'; + } else { + return 'unknown_os'; + } + } + + static validateRules(rules){ + if(rules == null) return true + + let result = true + rules.forEach(function(rule){ + const action = rule['action'] + const osProp = rule['os'] + if(action != null){ + if(osProp != null){ + const osName = osProp['name'] + const osMoj = Library.mojangFriendlyOS() + if(action === 'allow'){ + result = osName === osMoj + return + } else if(action === 'disallow'){ + result = osName !== osMoj + return + } + } + } + }) + return result + } +} + +class DLTracker { + constructor(dlqueue, dlsize){ + this.dlqueue = dlqueue + this.dlsize = dlsize + } +} + +class AssetGuard extends EventEmitter{ + constructor(){ + super() + this.totaldlsize = 0; + this.progress = 0; + this.assets = new DLTracker([], 0) + this.libraries = new DLTracker([], 0) + this.files = new DLTracker([], 0) + } +} + +//Instance of AssetGuard + +const instance = new AssetGuard() + +/* Utility Functions */ + +validateLocal = function(filePath, algo, hash){ + if(fs.existsSync(filePath)){ + let fileName = path.basename(filePath) + let shasum = crypto.createHash(algo) + let content = fs.readFileSync(filePath) + shasum.update(content) + let calcdhash = shasum.digest('hex') + return calcdhash === hash + } + return false; +} + +/* Validation Functions */ + +/** + * Load version asset index. + */ +loadVersionData = function(version, basePath, force = false){ + return new Promise(function(fulfill, reject){ + const name = version + '.json' + const url = 'https://s3.amazonaws.com/Minecraft.Download/versions/' + version + '/' + name + const versionPath = path.join(basePath, 'versions', version) + const versionFile = path.join(versionPath, name) + if(!fs.existsSync(versionFile) || force){ + //This download will never be tracked as it's essential and trivial. + request.head(url, function(err, res, body){ + console.log('Preparing download of ' + version + ' assets.') + mkpath.sync(versionPath) + const stream = request(url).pipe(fs.createWriteStream(versionFile)) + stream.on('finish', function(){ + fulfill(JSON.parse(fs.readFileSync(versionFile))) + }) + }) + } else { + fulfill(JSON.parse(fs.readFileSync(versionFile))) + } + }) +} + +/** + * Public asset validation method. + */ +validateAssets = function(versionData, basePath, force = false){ + return new Promise(function(fulfill, reject){ + assetChainIndexData(versionData, basePath, force).then(() => { + fulfill() + }) + }) +} + +//Chain the asset tasks to provide full async. The below functions are private. + +assetChainIndexData = function(versionData, basePath, force = false){ + return new Promise(function(fulfill, reject){ + //Asset index constants. + const assetIndex = versionData.assetIndex + const name = assetIndex.id + '.json' + const indexPath = path.join(basePath, 'assets', 'indexes') + const assetIndexLoc = path.join(indexPath, name) + + let data = null + if(!fs.existsSync(assetIndexLoc) || force){ + console.log('Downloading ' + versionData.id + ' asset index.') + mkpath.sync(indexPath) + const stream = request(assetIndex.url).pipe(fs.createWriteStream(assetIndexLoc)) + stream.on('finish', function() { + data = JSON.parse(fs.readFileSync(assetIndexLoc, 'utf-8')) + assetChainValidateAssets(versionData, basePath, data).then(() => { + fulfill() + }) + }) + } else { + data = JSON.parse(fs.readFileSync(assetIndexLoc, 'utf-8')) + assetChainValidateAssets(versionData, basePath, data).then(() => { + fulfill() + }) + } + }) +} + +assetChainValidateAssets = function(versionData, basePath, indexData){ + return new Promise(function(fulfill, reject){ + + //Asset constants + const resourceURL = 'http://resources.download.minecraft.net/' + const localPath = path.join(basePath, 'assets') + const indexPath = path.join(localPath, 'indexes') + const objectPath = path.join(localPath, 'objects') + + const assetDlQueue = [] + let dlSize = 0; + //const objKeys = Object.keys(data.objects) + async.forEachOfLimit(indexData.objects, 10, function(value, key, cb){ + const hash = value.hash + const assetName = path.join(hash.substring(0, 2), hash) + const urlName = hash.substring(0, 2) + "/" + hash + const ast = new Asset(key, hash, String(value.size), resourceURL + urlName, path.join(objectPath, assetName)) + if(!validateLocal(ast.to, 'sha1', ast.hash)){ + dlSize += (ast.size*1) + assetDlQueue.push(ast) + } + cb() + }, function(err){ + instance.assets = new DLTracker(assetDlQueue, dlSize) + instance.totaldlsize += dlSize + fulfill() + }) + }) +} + +/** + * Public library validation method. + */ +validateLibraries = function(versionData, basePath){ + return new Promise(function(fulfill, reject){ + + const libArr = versionData.libraries + const libPath = path.join(basePath, 'libraries') + + const libDlQueue = [] + let dlSize = 0 + + //Check validity of each library. If the hashs don't match, download the library. + async.eachLimit(libArr, 5, function(lib, cb){ + if(Library.validateRules(lib.rules)){ + let artifact = (lib.natives == null) ? lib.downloads.artifact : lib.downloads.classifiers[lib.natives[Library.mojangFriendlyOS()]] + const libItm = new Library(lib.name, artifact.sha1, artifact.size, artifact.url, path.join(libPath, artifact.path)) + if(!validateLocal(libItm.to, 'sha1', libItm.hash)){ + dlSize += (libItm.size*1) + libDlQueue.push(libItm) + } + } + cb() + }, function(err){ + instance.libraries = new DLTracker(libDlQueue, dlSize) + instance.totaldlsize += dlSize + fulfill() + }) + }) +} + +runQueue = function(){ + this.progress = 0; + let win = remote.getCurrentWindow() + + //Start asset download + let assetacc = 0; + const concurrentAssetDlQueue = instance.assets.dlqueue.slice(0) + async.eachLimit(concurrentAssetDlQueue, 20, function(asset, cb){ + mkpath.sync(path.join(asset.to, "..")) + let req = request(asset.from) + let writeStream = fs.createWriteStream(asset.to) + req.pipe(writeStream) + req.on('data', function(chunk){ + instance.progress += chunk.length + assetacc += chunk.length + instance.emit('assetdata', assetacc) + console.log('Asset Progress', assetacc/instance.assets.dlsize) + win.setProgressBar(instance.progress/instance.totaldlsize) + //console.log('Total Progress', instance.progress/instance.totaldlsize) + }) + writeStream.on('close', cb) + }, function(err){ + if(err){ + instance.emit('asseterr') + console.log('An asset failed to process'); + } else { + instance.emit('assetdone') + console.log('All assets have been processed successfully') + } + instance.totaldlsize -= instance.assets.dlsize + instance.assets = new DLTracker([], 0) + win.setProgressBar(-1) + }) + + //Start library download + let libacc = 0 + const concurrentLibraryDlQueue = instance.libraries.dlqueue.slice(0) + async.eachLimit(concurrentLibraryDlQueue, 1, function(lib, cb){ + mkpath.sync(path.join(lib.to, '..')) + let req = request(lib.from) + let writeStream = fs.createWriteStream(lib.to) + req.pipe(writeStream) + + req.on('data', function(chunk){ + instance.progress += chunk.length + libacc += chunk.length + instance.emit('librarydata', libacc) + console.log('Library Progress', libacc/instance.libraries.dlsize) + win.setProgressBar(instance.progress/instance.totaldlsize) + }) + writeStream.on('close', cb) + }, function(err){ + if(err){ + instance.emit('libraryerr') + console.log('A library failed to process'); + } else { + instance.emit('librarydone') + console.log('All libraries have been processed successfully'); + } + instance.totaldlsize -= instance.libraries.dlsize + instance.libraries = new DLTracker([], 0) + win.setProgressBar(-1) + }) +} + +module.exports = { + loadVersionData, + validateAssets, + validateLibraries, + runQueue, + instance +} \ No newline at end of file diff --git a/app/assets/js/library.js b/app/assets/js/library.js deleted file mode 100644 index f4a58f5b..00000000 --- a/app/assets/js/library.js +++ /dev/null @@ -1,37 +0,0 @@ - -exports.mojangFriendlyOS = function(){ - const opSys = process.platform - if (opSys === 'darwin') { - return 'osx'; - } else if (opSys === 'win32'){ - return 'windows'; - } else if (opSys === 'linux'){ - return 'linux'; - } else { - return 'unknown_os'; - } -} - -exports.validateRules = function(rules){ - if(rules == null) return true - - let result = true - rules.forEach(function(rule){ - const action = rule['action'] - const osProp = rule['os'] - if(action != null){ - if(osProp != null){ - const osName = osProp['name'] - const osMoj = exports.mojangFriendlyOS() - if(action === 'allow'){ - result = osName === osMoj - return - } else if(action === 'disallow'){ - result = osName !== osMoj - return - } - } - } - }) - return result -} \ No newline at end of file diff --git a/app/assets/js/script.js b/app/assets/js/script.js index 085bfa8e..80b375d9 100644 --- a/app/assets/js/script.js +++ b/app/assets/js/script.js @@ -1,13 +1,26 @@ var $ = require('jQuery'); const remote = require('electron').remote const shell = require('electron').shell +const path = require('path') /* Open web links in the user's default browser. */ $(document).on('click', 'a[href^="http"]', function(event) { event.preventDefault(); - shell.openExternal(this.href); + //testdownloads() + shell.openExternal(this.href) }); +testdownloads = async function(){ + const ag = require(path.join(__dirname, 'assets', 'js', 'assetguard.js')) + const basePath = path.join(__dirname, '..', 'mcfiles') + let versionData = await ag.loadVersionData('1.11.2', basePath) + await ag.validateAssets(versionData, basePath) + console.log('assets done') + await ag.validateLibraries(versionData, basePath) + console.log('libs done') + ag.runQueue() +} + /*Opens DevTools window if you type "wcdev" in sequence. This will crash the program if you are using multiple DevTools, for example the chrome debugger in VS Code. */ diff --git a/index.js b/index.js index 4e862438..eca68cce 100644 --- a/index.js +++ b/index.js @@ -17,19 +17,6 @@ function createWindow() { win.setMenu(null) - /*//Code for testing, marked for removal one it's properly implemented. - const assetdl = require('./app/assets/js/assetdownload.js') - const basePath = path.join(__dirname, 'mcfiles') - const dataPromise = assetdl.parseVersionData('1.11.2', basePath) - dataPromise.then(function(data){ - assetdl.downloadAssets(data, basePath) - //assetdl.downloadAssets(data, basePath) - //assetdl.downloadClient(data, basePath) - //assetdl.downloadLogConfig(data, basePath) - //assetdl.downloadLibraries(data, basePath) - //require('./app/assets/js/launchprocess.js').launchMinecraft(data, basePath) - })*/ - win.on('closed', () => { win = null })