.gitignore000066400000000000000000000001001516100555600130410ustar00rootroot00000000000000/build /node_modules /*.log .idea/workspace.xml .idea/tasks.xml .idea/000077500000000000000000000000001516100555600120425ustar00rootroot00000000000000.idea/misc.xml000066400000000000000000000002561516100555600135220ustar00rootroot00000000000000 .idea/modules.xml000066400000000000000000000004061516100555600142340ustar00rootroot00000000000000 .idea/tools.iml000066400000000000000000000005201516100555600137020ustar00rootroot00000000000000 .idea/vcs.xml000066400000000000000000000002471516100555600133620ustar00rootroot00000000000000 .npmignore000066400000000000000000000002221516100555600130550ustar00rootroot00000000000000/.idea /artifacts /build /test /node_modules /*.iml /*.ipr /*.iws /.travis.yml /.scrutinizer.yml /Gruntfile.js /*.lock *.log /corifeus-boot.json .scrutinizer.yml000066400000000000000000000007141516100555600142460ustar00rootroot00000000000000checks: javascript: true filter: excluded_paths: - test/* - node_modules/* - build/* - docs/* build: cache: disabled: true dependencies: before: - export LATEST=$(nvm ls-remote | tail -1) - nvm install $LATEST # - nvm use $LATEST - npm install -g grunt-cli tests: override: - command: 'grunt' coverage: file: 'build/coverage/clover.xml' format: 'clover' .travis.yml000066400000000000000000000001221516100555600131660ustar00rootroot00000000000000language: node_js node_js: - "node" before_script: - npm install grunt-cli -g Gruntfile.js000066400000000000000000000004541516100555600133620ustar00rootroot00000000000000module.exports = (grunt) => { const builder = require(`corifeus-builder`); const loader = new builder.loader(grunt); loader.js({ replacer: { type: 'p3x', npmio: true, }, }); grunt.registerTask('default', builder.config.task.build.js); };LICENSE000066400000000000000000000021411516100555600120650ustar00rootroot00000000000000MIT License Copyright (c) 2017 Patrik Laszlo / patrikx3 / https://patrikx3.com and contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. README.md000066400000000000000000000060251516100555600123440ustar00rootroot00000000000000[//]: #@corifeus-header [![Build Status](https://travis-ci.org/patrikx3/tools.svg?branch=master)](https://travis-ci.org/patrikx3/tools) [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/patrikx3/tools/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/patrikx3/tools/?branch=master) [![Code Coverage](https://scrutinizer-ci.com/g/patrikx3/tools/badges/coverage.png?b=master)](https://scrutinizer-ci.com/g/patrikx3/tools/?branch=master) [![NPM](https://nodei.co/npm/p3x-tools.png?downloads=true&downloadRank=true&stars=true)](https://www.npmjs.com/package/p3x-tools/) --- # Tools This is an open source project. Just code. ### Node Version Requirement ``` >=7.8.0 ``` ### Built on Node ``` v8.3.0 ``` The ```async``` and ```await``` keywords are required. Install NodeJs: https://nodejs.org/en/download/package-manager/ # Description [//]: #@corifeus-header:end This is an internal tool program (```p3x```) to manage a recursive async and serial command for NPM packages. # Examples All folders exclude ```node_modules``` and ```bower_components``` except it is included. ```bash p3x docker clean # delete images the have exited p3x docker clear # delete all images in docker p3x docker free # delete exited containers p3x pkg yarn install | # installs recursive every project that has a package.json p3x build yarn install | # install recursive every package that has a corifeus-builder p3x npm unpublish # unpublish every package from NPM except the last Minor version (Major.Minor.Build-Commit) p3x link # links every recursive given package together p3x git # you can execute a command on every git dir, recursive p3x rm # remove recursive the given folder p3x ncu -a # updates recursive all packages p3x git pull # recursive pull every folder p3x git push # recursive push every folder ``` If you want the commands is serials, add the ```-s``` or ```--serial``` flag, otherwise all ```async```. etc ... # Dependencies Fix http://cdn.corifeus.com/git/tools/dependencies-fix.json # Default excludes for CDN: http://cdn.corifeus.com/git/corifeus-app-web-pages/src/json/settings.json http://cdn.corifeus.com/git/tools/github.json # Github fork upstream 1. Clone your fork: ```bash git clone git@github.com:YOUR-USERNAME/YOUR-FORKED-REPO.git ``` 1. Add remote from original repository in your forked repository: ```bash cd into/cloned/fork-repo git remote add upstream git://github.com/ORIGINAL-DEV-USERNAME/REPO-YOU-FORKED-FROM.git git fetch upstream ``` 1. Updating your fork from original repo to keep up with their changes: ```bash git pull upstream master ``` # Warning / deprecate ```js npm deprecate [@] // or better npm deprecate my-thing@"< 0.2.3" "critical bug fixed in v0.2.3" ``` [//]: #@corifeus-footer --- [**P3X-TOOLS**](https://pages.corifeus.com/tools) Build v1.1.138-110 [Corifeus](http://www.corifeus.com) by [Patrik Laszlo](http://patrikx3.com) [//]: #@corifeus-footer:end bin/000077500000000000000000000000001516100555600116325ustar00rootroot00000000000000bin/p3x.js000077500000000000000000000023141516100555600127050ustar00rootroot00000000000000#!/usr/bin/env node if (!require('fs').existsSync(`${__dirname}/../node_modules`)) { require('child_process').execSync(`cd ${__dirname}/..; npm install`, { stdio: 'inherit' }); } const commander = require('commander'); const utils = require('corifeus-utils'); const pkg = require(`../package.json`); const mz = require('mz'); const start = async() => { // command // required // [command] optional // [command ...] variable options commander .version(pkg.version) .usage('[options]') ; require('../src/command/rm') require('../src/command/npm') require('../src/command/for') require('../src/command/git') require('../src/command/github') require('../src/command/each') require('../src/command/ncu') require('../src/command/docker') require('../src/command/server') require('../src/command/redis') // redis-cli KEYS chrome:* | xargs redis-cli DEL /* const isInModule = await mz.fs.exists(`${__dirname}/../../node_modules`); if (!isInModule) { require('../src/command/pdf') } */ commander.parse(process.argv); if (!process.argv.slice(2).length) { commander.outputHelp(); } } start();dependencies-fix.json000066400000000000000000000000251516100555600151640ustar00rootroot00000000000000{ "always": { } }github.json000066400000000000000000000003351516100555600132400ustar00rootroot00000000000000{ "excludes": [ "cookie", "jGrowl", "stackicons", "grunt-angular-templates", "electron-apps", "npm-check-updates", "node-v8-workers", "entware-packages-3x", "js-written-number" ] }package.json000066400000000000000000000022751516100555600133560ustar00rootroot00000000000000{ "name": "p3x-tools", "version": "1.1.138-110", "corifeus": { "prefix": "p3x-", "publish": true, "type": "p3x", "code": "Split" }, "bin": { "p3x": "bin/p3x.js" }, "license": "MIT", "description": "Tools", "main": "index.js", "directories": { "test": "test" }, "scripts": { "test": "grunt" }, "repository": { "type": "git", "url": "git+https://github.com/patrikx3/tools.git" }, "keywords": [ "tools", "unpublish", "npm" ], "author": "Patrik Laszlo ", "bugs": { "url": "https://github.com/patrikx3/tools/issues" }, "homepage": "https://pages.corifeus.com/tools", "devDependencies": { "corifeus-builder": "^1.7.735-81" }, "dependencies": { "commander": "^2.11.0", "corifeus-utils": "^1.1.413-102", "fs-extra": "^4.0.1", "github-api": "^3.0.0", "globby": "^6.1.0", "ini": "^1.3.4", "npm-check-updates": "^2.12.1", "progress": "^2.0.0", "tmp-promise": "^1.0.3" }, "engines": { "node": ">=7.8.0" } }src/000077500000000000000000000000001516100555600116515ustar00rootroot00000000000000src/command/000077500000000000000000000000001516100555600132675ustar00rootroot00000000000000src/command/docker.js000066400000000000000000000024711516100555600151000ustar00rootroot00000000000000const commander = require('commander'); const utils = require('corifeus-utils'); commander .command('docker ') .description(`Commands: free: deletes no-name images and dead containers clean: delete images the have exited clear: delete delete all images `) .action(async function (command, options) { switch (command) { case 'free': console.log('deletes no-name images and dead containers') await utils.childProcess.exec(`docker images --no-trunc | grep '' | awk '{ print $3 }' | xargs -r docker rmi || true docker ps --filter status=dead --filter status=exited -aq | xargs docker rm -v || true`, true) case 'clean': console.log('delete images the have exited') await utils.childProcess.exec(`docker rm $(docker ps -q -f status=exited) docker rmi $(docker images -q -f dangling=true) || true`, true) break; case 'clear': console.log('delete all images') await utils.childProcess.exec(`docker rm $(docker ps -a -q) docker rmi $(docker images -q)|| true`, true) break; default: console.error(`Unknown error: ${command}`) process.exit(1) break; } }) ; src/command/each.js000066400000000000000000000204661516100555600145350ustar00rootroot00000000000000const commander = require('commander'); const utils = require('corifeus-utils'); const find = utils.fs.find; const path = require('path'); const git = require('../git'); const mz = require('mz'); const npmLib = require('../npm'); const lib = require('../lib'); const _ = require('lodash'); const dependenciesFix = require('../../dependencies-fix.json'); const allCommands = [ 'link', 'publish', 'pkg', 'build', ]; const publishableCommand = [ 'link', 'publish' ]; const read = async (options, bar) => { if (options.read) { if (bar) { bar.interrupt(` Wait for enter... `) } await utils.input.key(); } } const loadCommander = (command) => { commander .command(`${command} [plusCommands...]`) .option('-d, --dry', 'Do not actually remove packages, just show what it does') .option('-a, --all', 'All') .option('-s, --serial', 'Serial ') .option('-r, --read', 'read a key') .option('-n, --non-interactive', 'Non interfactive') .option('--registry', 'Registry') .option('-m, --packageManager ', 'Package manager') .option('-o, --only ', 'Only packages', (list) => { return list.split(','); }) .action(async function (plusCommands, options) { await executeCommand(command, plusCommands, options); }) ; } allCommands.forEach((cmd) => loadCommander(cmd)) const getPkgAndDeps = async(file) => { const data = (await mz.fs.readFile(file)).toString(); try { const pkg = JSON.parse(data); let deps = Object.keys(Object.assign(pkg.dependencies || {}, pkg.devDependencies || {})); return [pkg, deps]; } catch(e) { console.error(); console.error(file); console.error(); throw e; } } const getNcu = (options) => { const exclude = dependenciesFix.always || {}; let excludeAddon = ''; const excludeKeys = Object.keys(exclude ); if (excludeKeys.length > 0) { excludeAddon = `-x ${excludeKeys.join(',')}` } return `ncu ${options.all ? '-a -u' : ''} --loglevel verbose --packageFile package.json ${excludeAddon}` } const executeCommand = async (command, plusCommands, options) => { let errors = []; plusCommands = plusCommands.join(' ').trim(); if (plusCommands === 'ncu') { plusCommands += ' --loglevel verbose --packageFile package.json' } if (options.nonInteractive) { plusCommands += ' --non-interactive' } if (options.registry) { plusCommands += ' --registry https://registry.npmjs.com/' } if (options.packageManager) { plusCommands += ' --packageManager ' + options.packageManager } if (options.all && plusCommands !== 'start') { plusCommands += ' -a' } let paths = await find({ find: 'package.json', }); let count = 0; const key = {}; let list = []; await paths.forEachAsync(async (findData) => { const [pkg, deps] = await getPkgAndDeps(findData.path); key[pkg.name] = true; const result = { name: pkg.name, pkg: pkg, deps: deps, findData: findData } list.push(result) }); list = list.map((item) => { item.wants = []; item.deps.forEach((want) => { if (key.hasOwnProperty(want)) { item.wants.push(want); } }) return item; }) list = utils.require.resovleDependencies({ modules: list, debug: false, recursive: [ 'corifeus-utils' ] }); if (options.only !== undefined) { list = list.filter(item => { return options.only.includes(item.pkg.name); }) } const allList = list.slice(); if (publishableCommand.includes(command)) { list = list.filter(item => { return item.pkg.hasOwnProperty('corifeus') && item.pkg.corifeus.publish === true; }) } switch(command) { case 'publish': options.serial = true; break; case 'link': plusCommands = `yarn unlink || true yarn link `; break; } if (plusCommands === '') { plusCommands = 'list'; } if (plusCommands === 'start') { plusCommands = `rm -rf node_modules ${getNcu({all: true})} yarn install --non-interactive ${npmLib.command.publish({ all: options.all } )}`; } const actual = []; let doActualExecute = false; const displayCommand = `${command} ${plusCommands}`; const bar = lib.newProgress(command, list); let remained = []; await list.forEachAsync(async (item) => { const {findData , pkg, deps} = item; let hasBuilder; if (pkg.name !== undefined && pkg.name.startsWith('corifeus-builder')) { hasBuilder = true; } else if (command === 'build' || command === 'publish') { hasBuilder = deps.find((dep) => { return dep.startsWith('corifeus-builder'); }) } else { hasBuilder = true; } if (hasBuilder !== undefined ) { actual.push(item); switch (plusCommands) { case 'count': case 'deps': case 'list': console.info(pkg.name) utils.repeat(2, () => { bar.tick({ token: pkg.name }) }) break; default: if (!options.dry) { doActualExecute = true; } await lib.executeCommandByPath({ findData: findData, command: plusCommands, errors: errors, item: item, options: options, bar : bar }) await read(options, bar) } } else { utils.repeat(2, () => { bar.tick({ token: pkg.name }) }) remained.push(item); } }, options.serial) remained.forEach(allItem => { let found = false; actual.forEach((actualItem) => { if (actualItem.name === allItem.name) { found = true; } }) if (!found) { remained.push(allItem); } }, options.serial) if ((doActualExecute || options.dry) && publishableCommand.includes(command)) { const afterBar = lib.newProgress(`post ${command}`, list); await allList.forEachAsync(async (item) => { const {findData , pkg, deps} = item; let execCommand; switch(command) { case 'publish': execCommand = ` ${getNcu({all: true})} rm -rf ./node_modules yarn install `; break; case 'link': if (item.wants.length > 0) { execCommand = ` yarn link ${item.wants.join(' \nyarn link ')} ` } break; } if (execCommand !== undefined) { await lib.executeCommandByPath({ findData: findData, command: execCommand, errors: errors, item: item, options: options, bar : afterBar }) await read(options, afterBar) } }, options.serial ) } console.info(`All: ${allList.map((item) => item.name)}`) console.info(); console.info(`Actual: ${actual.map((item) => item.name)}`) console.info(); console.info(`Remained: ${remained.map((item) => item.name)}`) console.info(); console.info(`Actual count: ${actual.length}`) console.info(); console.info(`Serial: ${options.serial ? 'true' : 'false'}`) if (errors.length > 0) { console.error(`Errors: ${errors.length}`); console.error(errors) } console.info(); console.info(displayCommand) console.info(); console.info('Dependencies fix', JSON.stringify(dependenciesFix, null, 4)) }src/command/for.js000066400000000000000000000040431516100555600144140ustar00rootroot00000000000000// rmdirr dir dirs const commander = require('commander'); const fsExtra = require('fs-extra'); const utils = require('corifeus-utils'); commander .command('for ') .description(` Finds a list of directories (without start ./ and end /) recursively `) .option('-d, --dry', 'Do not actually remove packages, just show what it does') .option('-t, --types ', 'Defaults is all, options: d = directory, f = file, like -t=f,d', (val) => { return val.split(','); }) .option('-x, --exclude ', 'Exclude paths, default is node_module', (val) => { const types = {}; val.split(',').forEach((type) => types[type] = true); }) .action(async function (dir, command, options) { command = command.join(' '); const find = utils.fs.find; console.info(`Directory finding: ${dir}`) const paths = await find({ find: dir, type: options.types || { d: true, f: true } }); const dry = options.dry || false; if (dry) { console.info(`Dry, doesn't do anything, just shows what it does`) } const promises = []; paths.forEach(async (findData) => { const generatedCommand = `bash -c ' pushd ${findData.dir} set -e export FOUND_DIR=${findData.dir} export FOUND=${findData.path} ${command} popd '`; if (dry) { console.info(`Dirs`, path) // console.log(generatedCommand); } else { const run = utils.childProcess.exec(generatedCommand); run.exec.stdout.on('data', (data) => { console.info(data); }); run.exec.stderr.on('data', (data) => { console.error(data); }); promises.push(run); } //console.info(`Path ${path}, Execute ${command}`) }) Promise.all(promises); }) ;src/command/git.js000066400000000000000000000066301516100555600144150ustar00rootroot00000000000000const commander = require('commander'); const utils = require('corifeus-utils'); const find = utils.fs.find; const path = require('path'); const git = require('../git'); const mz = require('mz'); const lib = require('../lib'); //p3x for .git 'cd ..;p3x git truncate' const commands = [ 'truncate', 'renew', 'each', 'push', 'pull', ] commander .command('git [command] [plusCommands...]') .description(` The versioning is Major.Minor.Commit-Build If you omit the package name, it will use all. commands: ${commands.join(', ')} `) .option('-d, --dry', 'Do not actually remove packages, just show what it does') .option('-a, --all', 'All') .option('-s, --serial', 'Serial') .action(async function (command, plusCommands, options) { let paths; if (command === undefined) { command = 'list'; } plusCommands = plusCommands.join(' ').trim(); switch(command) { case 'truncate': case 'renew': const truncate = require('../git').truncate; await truncate(options); break; case 'each': case 'count': case 'list': case 'push': case 'pull': paths = await find('.git'); paths = paths.map(pathActual => { if (pathActual.dir === pathActual.path) { pathActual.dir = path.dirname(pathActual.dir); } return pathActual; }) let internalCommand; switch (command) { case 'count': console.info(paths.length); break; case 'each': break; case 'list': console.log(path.basename(findData.dir)); break; case 'push': await paths.forEachAsync(async (findData) => { await utils.repeat.async(2, async() => { await lib.executeCommandByPath({ findData: findData, options: options, command: `git add . git commit -am 'p3x-robot-push' || true git push || true ${plusCommands === '' ? 'true' : plusCommands}`, }) }, true) }, true) break; case 'pull': internalCommand = `git pull` break; } if (internalCommand !== undefined) { let errors = []; const bar = lib.newProgress(command, paths) await paths.forEachAsync(async (findData) => { await lib.executeCommandByPath({ findData: findData, command: internalCommand, options: options, errors: errors, bar: bar, }) }, options.serial) } break; default: console.error(`Unknown command: ${command}`) } }) ; src/command/github.js000066400000000000000000000047641516100555600151220ustar00rootroot00000000000000const commander = require('commander'); const utils = require('corifeus-utils'); const find = utils.fs.find; const path = require('path'); const github = require('../github'); const defaultGithubExcludes = require('../../github.json').excludes //p3x for .git 'cd ..;p3x git truncate' const commands = [ 'mirror', 'pull', ] commander .command('github ') .description(` commands: ${commands.join(', ')} `) .option('-d, --dry', 'Do not actually remove packages, just show what it does') .option('-u, --user [user]', 'The GitHub repo, default is patrikx3') .option('-n, --note [note]', 'The note in the commit, the default is p3x-robot sunday release') .option('-p, --password [password]', 'The GitHub password or token') .option('-b, --branch ', 'The default branch is master') .option('-g, --git ', 'The GIT repo, example is https://user:password@git.patrikx3.com/') .option('-x, --exclude ', `Exclude paths, default is ${defaultGithubExcludes.join(',')}`, (val) => { return val.split(','); }) .option('-o, --only ', `Only included paths, default is all`, (val) => { return val.split(','); }) .action(async function (command, options) { const user = options.user || 'patrikx3'; const gitUrl = options.git || 'https://git.patrikx3.com' ; const note = options.note || 'p3x-robot sunday release' const branch = options.branch || 'master'; const exclude = options.exclude || defaultGithubExcludes console.info(`User: ${user}`); console.info(`Git url: ${gitUrl}`); console.info(`Note: ${note}`); switch(command) { case 'pull': await utils.childProcess.exec(` GIT_NAME=$(basename \`git rev-parse --show-toplevel\`) git pull https://github.com/patrikx3/$GIT_NAME ${branch}`, true); break; case 'mirror': if (options.password === undefined) { throw new Error('password required') } await github.mirror({ user: user, password: options.password, gitUrl: gitUrl, dry: options.dry, note: note, exclude: exclude, only: options.only }) break; default: console.error(`Unknown command: ${command}`) } }) ; src/command/ncu.js000066400000000000000000000007021516100555600144110ustar00rootroot00000000000000const commander = require('commander'); const utils = require('corifeus-utils'); const path = require('path'); const mz = require('mz'); const lib = require('../lib'); commander .command('ncu') .description(`This is for global update`) .action(async function (options) { const ncu = require('npm-check-updates'); const result = await ncu.run({ global: true, }) console.log(result); }) ; src/command/npm.js000066400000000000000000000052221516100555600144200ustar00rootroot00000000000000const commander = require('commander'); const utils = require('corifeus-utils'); const mz = require('mz'); const hasin = require('lodash/hasIn'); const globby = require('globby'); const npmLib = require('../npm'); const commands = [ 'publish', 'update', 'login', 'unpublish', ] commander .command('npm [packages...]') .description(` The versioning is Major.Minor.Commit-Build If you omit the package name, it will use all. commands: ${commands.join(', ')} `) .option('-u, --username [username]', 'Author username, defaults to patrikx3', (val) => { return val.split(','); }) .option('-s, --search [term]', `Search for the given packages, the default is 'p3x,corifeus`, (val) => { return val.split(','); }) .option('-a, --all') .option('-d, --dry', 'Do not actually remove packages, just show what it does') .action(async function (command, packages, options) { let search = options.search; if (search === undefined) { search = ['p3x', 'corifeus'] } let username = options.username; if (username === undefined) { username = ['patrikx3']; } const dry = options.dry || false; const all = options.all || false; switch(command) { case 'update': if (await mz.fs.exists('./package.json')) { await utils.childProcess.exec(`ncu -a --loglevel verbose --packageFile package.json`, true) } break; case 'unpublish': const unpublish = require('../npm-unpublish'); await unpublish(username, search, packages, dry, all); break; case 'publish': if (await mz.fs.exists('./package.json')) { const pkg = require(`${process.cwd()}/package.json`); if (hasin(pkg, 'corifeus.publish') && pkg.corifeus.publish === true) { await utils.childProcess.exec(npmLib.command.publish({ all: options.all }), true) } else { console.info(`This package.json has not corifeus.publish = true`); } } else { console.info(`This directory has no package.json`); } break; case 'login': await utils.childProcess.exec(`npm login --registry https://registry.npmjs.org`, true) break; default: console.error(`Unknown command: ${command}`) } }) ; src/command/pdf.js000066400000000000000000000005261516100555600144010ustar00rootroot00000000000000/* const commander = require('commander'); // unpublish commander .command('pdf [file]') .description(` Parse a pdf file `) .option('-d, --dry', 'Do not actually remove packages, just show what it does') .action(async function (file, options) { const pdf = require('../pdf'); await pdf(file); }) ; */src/command/redis.js000066400000000000000000000024431516100555600147360ustar00rootroot00000000000000const commander = require('commander'); const utils = require('corifeus-utils'); const path = require('path'); const mz = require('mz'); const lib = require('../lib'); commander .command('redis [command] [param]') .option('-u, --url ', 'REDIS url') .description(`This is for redis: DEL `) .action(async function (command, param, options) { let addon = ''; if (options.url) { console.log(options.url) const url = new (require('url').URL)(options.url) if (url.hostname) { addon += `-h ${url.hostname} ` } if (url.password) { addon += `-a ${url.password} ` } const pathname = url.pathname.substr(1); if (pathname != '') { addon += `-n ${pathname} ` } } switch (command.toLowerCase()) { case 'del': if (param === undefined) { throw new Error('DEL require a parameter') } const commandExec = `/usr/bin/redis-cli ${addon}KEYS ${param} | xargs redis-cli ${addon}DEL`; //console.info(commandExec); await utils.childProcess.exec(commandExec, true) break; } }) ; src/command/rm.js000066400000000000000000000016511516100555600142460ustar00rootroot00000000000000const commander = require('commander'); const fsExtra = require('fs-extra'); const utils = require('corifeus-utils'); commander .command('rm ') .description(` Delete the list of directories (without start ./ and end /) recursively `) .option('-d, --dry', 'Do not actually remove packages, just show what it does') .action(async function (dirs, options) { const find = utils.fs.find; const files = await find({ find: dirs, type: { d: true } }); const dry = options.dry || false; if (dry) { console.info(`Dry, doesn't remove anything`) } const promises = []; files.forEach(async (file) => { if (!dry) { promises.push(fsExtra.remove(file.path)); } console.info(`Delete ${file.path}`) }) Promise.all(promises); }) ;src/command/server.js000066400000000000000000000017551516100555600151430ustar00rootroot00000000000000const commander = require('commander'); const utils = require('corifeus-utils'); const path = require('path'); const mz = require('mz'); const lib = require('../lib'); commander .command('server [command] [param]') .description(`This is for global update`) .option(`-p, --password `) .action(async function (command, param, options) { let url; switch (command) { case 'scale': case 'worker': if (param === undefined) { param = 'auto'; } url = `http://localhost:23502/worker/${param}`; break; default: url = `http://localhost:23502/status`; break; } const response = await utils.http.request({ url: url, headers: { 'Authorization': options.password || '' } }); console.log(JSON.stringify(response.body, null, 2)); }) ; src/git.js000066400000000000000000000014131516100555600127710ustar00rootroot00000000000000const utils = require('corifeus-utils'); const globby = require('globby'); const path = require('path'); const truncate = async (options) => { const command = `git config --global credential.helper 'cache --timeout 7200' git checkout --orphan temp git add -A git commit -am "p3x-robot" git branch -D master git branch -m master git branch --set-upstream-to origin/master master git push -f origin master` console.log(command); if (!options.dry) { await utils.childProcess.exec(command, true) } } const findModules = async(root) => { const modules = await globby(`${root}/**/.gitmodules`); return modules.map((dir) => { return path.dirname(dir); }); } module.exports.findModules = findModules; module.exports.truncate = truncate; src/github.js000066400000000000000000000123141516100555600134720ustar00rootroot00000000000000const GitHub = require('github-api'); const tmp = require('tmp-promise'); const utils = require('corifeus-utils'); const globby = require('globby'); const mz = require('mz'); const ini = require('ini'); const url = require('url'); const git = require('./git'); const list = async(options) => { const {only, user , exclude } = options; const gh = new GitHub(); const response = await gh.getUser(user).listRepos(); const repos = response.data return repos.filter(repo => { let isOnly; if (only !== undefined) { isOnly = only.includes(repo.name) } else { isOnly = true; } return repo.full_name === `${user}/${repo.name}` && !exclude.includes(repo.name) && isOnly; }); } const mirror = async(options) => { let { only, user, password, gitUrl, dry, note, exclude } = options; let tmpDir; const errors = []; try { tmpDir = await tmp.dir(); console.info(`Generate tmp file: ${tmpDir.path}`); const repos = await list({ user: user, exclude: exclude, only: only }); await repos.forEachAsync(async(repo) => { await utils.childProcess.exec(` git clone https://${user}:${password}@github.com/${user}/${repo.name} ${tmpDir.path}/github/${repo.name} cd ${tmpDir.path}/github/${repo.name} git submodule update --init --recursive --remote git clone ${gitUrl}/${repo.name}.git ${tmpDir.path}/git/${repo.name} cd ${tmpDir.path}/git/${repo.name} git submodule update --init --recursive --remote `, true) }) console.info('Remove all Git .git dirs and move to Github .git dirs'); await repos.forEachAsync(async(repo) => { const currentRepo = `${tmpDir.path}/git/${repo.name}`; await utils.childProcess.exec(` rm -rf ${currentRepo}/.git rm -rf ${currentRepo}/yarn.lock rm -rf ${currentRepo}/package-lock.json mv ${tmpDir.path}/github/${repo.name}/.git ${tmpDir.path}/git/${repo.name}/ `, true) }) console.info('Move GIT to Github dir'); await utils.childProcess.exec(` mv ${tmpDir.path}/github/ ${tmpDir.path}/github-old/ mv ${tmpDir.path}/git/ ${tmpDir.path}/github/ `, true) console.info('Move submodules to GitHub'); await replaceGitSubmodules(`${tmpDir.path}/github`, user); console.info('Move init.sh to GitHub'); await replaceInitSh(`${tmpDir.path}/github`, gitUrl, user); await repos.forEachAsync(async(repo) => { await utils.childProcess.exec(` cd ${tmpDir.path}/github/${repo.name} pwd rm -rf ./idea git add . git status `, true) }) await repos.forEachAsync(async(repo) => { try { await utils.childProcess.exec(` cd ${tmpDir.path}/github/${repo.name} git commit -am "${note} ${new Date().toLocaleString()}" ${dry ? 'true' : 'git push'} `, true) } catch(e) { errors.push(e); } }, true) const modules = await git.findModules(`${tmpDir.path}/github/`) await modules.forEachAsync(async(module) => { await utils.childProcess.exec(` cd ${module} git pull git checkout master git submodule update --init --recursive --remote git submodule foreach --recursive git checkout master git status ${dry ? 'true' : 'git push'} `, true) }, true) console.info('All done') } catch(e ) { errors.push(e); } finally { if (tmpDir) { console.info(`Cleanup tmp file ${tmpDir.path}`); if (!dry) { await utils.childProcess.exec(`rm -rf ${tmpDir.path}`,true) } else { console.info('Dry, not cleanup') } } errors.forEach(e => { console.error(e) }) } } const replaceGitSubmodules = async(root, user) => { const files = await globby(`${root}/**/.gitmodules`) await files.forEachAsync(async(file) => { console.info(`submodule found ${file}`) const string = (await mz.fs.readFile(file)).toString(); const iniFile = ini.parse(string); Object.keys(iniFile).forEach((key) => { const submodule = iniFile[key] submodule.url = `https://github.com/${user}/${submodule.path}`; }) const result = ini.stringify(iniFile); await mz.fs.writeFile(file, result); console.info(`submodule replaced ${file}`) }) } const replaceInitSh = async(root, gitUrl, user) => { const gitUrlObj = url.parse(gitUrl); gitUrl = RegExp.escape(`${gitUrlObj.protocol}//${gitUrlObj.hostname}`); const gitUrlRegexp = new RegExp(gitUrl, 'ig') const files = await globby(`${root}/**/init.sh`) await files.forEachAsync(async(file) => { console.info(`init.sh found ${file}`) const string = (await mz.fs.readFile(file)).toString(); const result = string.replace(gitUrlRegexp, `https://github.com/${user}`) await mz.fs.writeFile(file, result); console.info(` ${result} `); console.info(`init.sh replaced ${file}`) }) } module.exports.list = list; module.exports.mirror = mirror; module.exports.replaceGitSubmodules = replaceGitSubmodules; module.exports.replaceInitSh = replaceInitSh;src/lib.js000066400000000000000000000041511516100555600127560ustar00rootroot00000000000000const utils = require('corifeus-utils'); const progress = require('progress'); const executeCommandByPath = async (options) => { const { findData, command, errors, bar } = options; const name = options.item ? options.item.name : command; const token = `${name} ${command}`; if (options.options.dry) { console.info('------------------------------------'); console.info(findData.path); console.info(name); console.info(command) console.info(); if (bar) { utils.repeat(2, () => { bar.tick({ token: token }) }) } return; } try { if (bar) { bar.tick({ token: token }) } const execPromise = utils.childProcess.exec(` bash -c ' pushd ${findData.dir} set -e export FOUND_DIR=${findData.dir} export FOUND=${findData.path} ${command} popd ' `, bar === undefined ? true : false) execPromise.exec.stdout.on('data', (data) => { if (bar) { bar.interrupt(data) } if (!bar) { console.log(data); } }); execPromise.exec.stderr.on('data', (data) => { if (bar) { bar.interrupt(data) } else { console.error(data); } }); await execPromise; } catch (e) { if (errors !== undefined) { errors.push(e); } throw e; } finally { if (bar) { bar.tick({ token: name }) } } } const newProgress = (status, list) => { const bar = new progress(`${status}[:bar] :token`, { total: list.length * 2, width: Math.min(list.length * 2, 20) , complete: '=', incomplete: '-', clear: true }) bar.tick(0, { token: 'loading' }) return bar; } module.exports.newProgress = newProgress; module.exports.executeCommandByPath = executeCommandByPath;src/npm-unpublish.js000066400000000000000000000112161516100555600150110ustar00rootroot00000000000000const utils = require('corifeus-utils') const path = require('path') // npm deprecate ${repo}@${version} "It is deprecated" const exec = require('child_process').exec; const repo = process.argv[2]; const removeVersion = (repo, version, keptVersions, dry, all) => { const semVersion = version.split('.'); if (!all) { const keptVersion = `${semVersion[0]}.${semVersion[1]}` if (!keptVersions.hasOwnProperty(keptVersion)) { console.info(`Kept version ${version}`); keptVersions[keptVersion] = version; return; } } const command = `npm unpublish ${repo}@${version} --registry https://registry.npmjs.org`; if (dry) { console.info(`[DRY] Removed version ${version}`); console.info(command) return; } return new Promise((resolve, reject) => { const run = exec(command, (e, stdout, stderr) => { if (e) { return reject(e); } if (stderr !== '') { return reject(new Error(stderr)); } console.info(`Removed version ${version}`); return resolve(stdout); }) run.stdout.on('data', (data) => { console.info(data); }); run.stderr.on('data', (data) => { console.error(data); }); }) } const findVersions = async (repo) => { return new Promise((resolve, reject) => { exec(`npm show ${repo} --json`, async(e, stdout, stderr) => { if (e) { console.error(e); reject(e); return; } const info = JSON.parse(stdout); const versions = info.versions.reverse(); console.info(`Versions:`, `${versions.length} versions`, versions); resolve(versions) }); }) } const removePackage = async (repo, dry, all) => { console.info(`Remove repo ${repo}`); const versions = await findVersions(repo); const keptVersions = {} if (Object.keys(versions).length > 1) { for(let version of versions) { await removeVersion(repo, version, keptVersions, dry, all); } } const keptVersionsArray = Object.values(keptVersions); const log = `${repo} Total: ${versions.length} Kept Versions: ${keptVersionsArray.length} Remained versions, ${keptVersionsArray.join(' , ')} `; console.info(log); return { versions: versions, kept: keptVersionsArray, log: log }; } module.exports = async (usernames, search, packages, dry, all) => { const isAll = packages.length === 0; console.info(` ---------------------------------------------- Unpublish ---------------------------------------------- Usernames: ${usernames.join(', ')} Search: ${search.join(', ')} Packages: ${isAll ? 'all' : packages.join(', ')} Dry: ${dry} `) const logs = []; const errorLogs = []; const promises = []; const createLog = (repo, dry) => { return new Promise(async(resolve, reject) => { try { const log = await removePackage(repo, dry, all); logs.push(log); resolve(); } catch(e) { reject(e); } }) } //https://registry.npmjs.org/-/v1/search?text=p3x if (isAll) { const searchPromises = []; search.forEach((term) => { searchPromises.push( utils.http.request(`https://registry.npmjs.org/-/v1/search?text=${term}`) ) }) const results = await Promise.all(searchPromises); // find let objects = []; results.forEach(result => { objects = objects.concat(result.body.objects) }) objects = objects .filter((obj) => { if (!obj.package.hasOwnProperty('author')) { errorLogs.push({ pkg: obj.package, log: `000-This package is invalid: ${obj.package.name} ${obj.package.version} ${JSON.stringify(obj.package, null, 2)} ` }) return false; } return usernames.includes(obj.package.author.username); }) objects.forEach((object) => { promises.push(createLog(object.package.name, dry)) }); } else { packages.forEach((pkg) => { promises.push(createLog(pkg, dry)) }) } await Promise.all(promises); logs.forEach((log) => { console.info(log.log); }) errorLogs.forEach((log) => { console.info(log.log); }) console.info(` Total: ${logs.length} Errors: ${errorLogs.length} `) return logs; } src/npm.js000066400000000000000000000003751516100555600130060ustar00rootroot00000000000000module.exports.command = { publish: (options = {all: true}) => `grunt cory-npm cory-replace grunt cory-npm-angular || true grunt publish -v || true sleep 5 npm publish ${options.all ? 'npm publish --registry https://registry.npmjs.org' : ''}` }src/old-lib.js000066400000000000000000000212541516100555600135350ustar00rootroot00000000000000/* const progress = require('progress'); const fs = require('mz/fs'); const path = require('path') const commander = require('commander'); const utils = require('corifeus-utils'); const execRaw = require('child_process').exec; const pkg = require('../package.json'); let loadModules = () => { const srcpath = `${process.cwd()}` const modules = fs.readdirSync(srcpath).filter(file => fs.statSync(path.join(srcpath, file)).isDirectory()).filter(dir => dir.startsWith('corifeus-')); modules.unshift('corifeus'); return modules; } const allProjects = loadModules() let projects; const projectPath = (project) => { return `${__dirname}/../${project === 'corifeus' ? '' : project + '/' }`; } const getPkg = async (project) => { const buffer = await fs.readFile(`${projectPath(project)}package.json`); return JSON.parse(buffer.toString()); } const extractPackages = async function (projects = allProjects) { const pkgs = {}; await utils.array.forEachAsync(projects, async (project) => { pkgs[project] = await getPkg(project) }, true) return pkgs; }; let pkgs; // order is important const settings = { 'corifeus-utils': { publish: true, }, 'corifeus-builder': { publish: true, }, 'corifeus-server': { publish: true, }, 'corifeus-server-deployer': { publish: true, }, 'corifeus-builder-angular': { publish: true, }, 'corifeus-web': { publish: true, }, 'corifeus-web-material': { publish: true, }, } const loadPublishableChildren = () => { Object.keys(settings).forEach((publishable) => { const setting = settings[publishable]; const children = {}; Object.keys(pkgs).forEach((pkgName) => { const pkg = pkgs[pkgName]; const updateDependecies = (repo) => { if (pkg.hasOwnProperty(repo) && pkg[repo].hasOwnProperty(publishable)) { if (Object.keys(pkg[repo]).includes(publishable)) { children[pkgName] = true; } } } updateDependecies('dependencies', publishable); updateDependecies('devDependencies', publishable); setting.children = Object.keys(children); }) }) } const init = async (commanderEnabled = true) => { pkgs = await extractPackages() if (commanderEnabled) { commander .version(pkg.version) .usage('[options]') .option('-v, --verbose', 'Verbose') .option('-n, --npm', 'Npm to original registry') .option('-p, --projects ', 'The list of projects', (val) => { return val.split(','); }) .parse(process.argv); } projects = commander.projects || allProjects loadPublishableChildren(); module.exports.projects = projects } const exec = (project, command, bar) => { return new Promise((resolve, reject) => { const actualCommand = `${project === 'corifeus' ? '' : `pushd ${project}`} ${command} ${project === 'corifeus' ? '' : `popd`} ` if (bar) { bar.interrupt(actualCommand); } else { console.log(actualCommand); } const run = execRaw(`bash -c ' set -e ${actualCommand} '`, { stdio: `inherit`, }, (e) => { if (e) { console.error(e); return reject(e); } if (bar) { bar.tick({ token: `${project}` }) } return resolve(); }) run.stdout.on('data', (data) => { if (commander.verbose && bar) { bar.interrupt(data) } if (!bar) { console.log(data); } }); run.stderr.on('data', (data) => { if (bar) { bar.interrupt(data) } else { console.error(data); } }); }) } const newProgress = (status, total = projects.length) => { const bar = new progress(`${status} [:bar] :token`, { total: total, width: 27, clear: true }) bar.tick(0, { token: `${projects[0]}` }) return bar; } const grunt = async () => { let port = 10000; const promises = []; const barGrunt = newProgress('grunt'); await utils.array.forEachAsync(projects, (project) => { promises.push(exec(project, `grunt --cory-port=${++port}`, barGrunt)); }) await Promise.all(promises); } const publish = async () => { let total = projects.length; const upgradeAbleCount = await getUpgradeAbleCount(); total = total + (total * upgradeAbleCount); const barPublish = newProgress('publish-upgrade', total); await utils.array.forEachAsync(projects, async (project) => { const setting = settings[project]; if (setting !== undefined && setting.hasOwnProperty('publish') ) { await exec(project, `${commander.npm ? '' : 'grunt cory-npm'} npm publish ${commander.npm ? ' --registry https://registry.npmjs.org' : ''}`, barPublish); await upgrade(barPublish) } else { barPublish.tick({ token: '...' }) } }, true) } const getUpgradable = async () => { const upgradeAble = {}; await utils.array.forEachAsync(projects, (project) => { const setting = settings[project]; if (setting !== undefined && setting.hasOwnProperty('publish') && setting.hasOwnProperty('children')) { setting.children.forEach((child) => { upgradeAble[child] = upgradeAble[child] || []; upgradeAble[child].push(project) }) } }) return upgradeAble; } const getUpgradeAbleCount = async() => { const upgradeAble = await getUpgradable(); const upgrades = Object.keys(upgradeAble); let total = upgrades.length; upgrades.forEach(() => total++); return total; } const upgrade = async (progresser) => { const upgradeAble = await getUpgradable(); const upgrades = Object.keys(upgradeAble); if (progresser === undefined) { const total = await getUpgradeAbleCount(); progresser = progresser || newProgress('upgrade', total); } const promises = []; await utils.array.forEachAsync(upgrades, async (project) => { progresser.tick({ token: project }) const pkgFile = `${projectPath(project)}package.json`; const pkg = await getPkg(project); await utils.array.forEachAsync(upgradeAble[project], async(linkUpdate) => { const updateDependecies = async (repo) => { if (pkg.hasOwnProperty(repo) && pkg[repo].hasOwnProperty(linkUpdate)) { const linkPkg = await getPkg(linkUpdate); pkg[repo][linkUpdate] = `^${linkPkg.version}`; } } await updateDependecies('dependencies', linkUpdate); await updateDependecies('devDependencies', linkUpdate); promises.push(fs.writeFile(pkgFile, JSON.stringify(pkg, null, 4))); progresser.interrupt(`${project} => new pkg ${linkUpdate}`); progresser.tick({ token: linkUpdate }) }) }) await Promise.all(promises); } const link = async() => { const upgradeAble = await getUpgradable(); const upgradeAbleKeys = Object.keys(upgradeAble); const projectsGenerated = projects.filter((project) => (settings[project] || {}).hasOwnProperty('children') ); let total = projectsGenerated.length + upgradeAbleKeys.length; const progress = newProgress('link', total); let promises = []; await utils.array.forEachAsync(projectsGenerated, (project) => { const setting = settings[project] || {}; if (setting.hasOwnProperty('publish') && setting.hasOwnProperty('children')) { promises.push( exec(project, `yarn unlink || true yarn link`, progress) ) } }) await Promise.all(promises); promises = []; await utils.array.forEachAsync(upgradeAbleKeys , (project) => { let command = `` upgradeAble[project].forEach((link) => { command += `yarn link ${link} `; }) promises.push(exec(project, command, progress)) }) await Promise.all(promises); } module.exports = { projects: projects, settings: settings, exec: exec, projectPath: projectPath, newProgress: newProgress, grunt: grunt, publish: publish, upgrade: upgrade, link: link, extractPackages: extractPackages, loadModules: loadModules, init: init } */src/pdf.js000066400000000000000000000026451516100555600127670ustar00rootroot00000000000000//const pdfjs = require('pdfjs-dist'); //const fsExtra = require('fs-extra'); const mz = require('mz'); const pdf = async(file) => { const buffer = await mz.fs.readFile(file); const doc = await pdfjs.getDocument(buffer); var numPages = doc.numPages; console.log('# Document Loaded'); console.log('Number of Pages: ' + numPages); console.log(); const meta = await doc.getMetadata() console.log('# Metadata Is Loaded'); console.log('## Info'); console.log(JSON.stringify(meta.info, null, 2)); console.log(); if (meta.metadata) { console.log('## Metadata'); console.log(JSON.stringify(meta.metadata.metadata, null, 2)); console.log(); } for (var pageNum = 1; pageNum <= numPages; pageNum++) { const page = await doc.getPage(pageNum); console.log('# Page ' + pageNum); var viewport = page.getViewport(1.0 /* scale */); console.log('Size: ' + viewport.width + 'x' + viewport.height); console.log(); const content = await page.getTextContent(); // Content contains lots of information about the text layout and // styles, but we need only strings at the moment var strings = content.items.map(function (item) { return item.str; }); console.log('## Text Content'); console.log(strings.join(' ')); // console.log(content); } } module.exports = pdf;test/000077500000000000000000000000001516100555600120415ustar00rootroot00000000000000test/scripts/000077500000000000000000000000001516100555600135305ustar00rootroot00000000000000test/scripts/find-modules.js000077500000000000000000000002771516100555600164650ustar00rootroot00000000000000#!/usr/bin/env node const path = '/tmp/tmp-6503ODo7z962XubX/github'; const git = require('../../src/git'); const start = async() => { console.log(await git.findModules(path)); } start();test/scripts/transform-git-initsh.js000077500000000000000000000002751516100555600201650ustar00rootroot00000000000000#!/usr/bin/env node const path = '/tmp/tmp-22337ZMnl1XYmwIs4/github'; const github = require('../../src/github'); github.replaceInitSh(path, 'https://123:121@git.patrikx3.com', 'patrikx3');test/scripts/transform-git-modules.js000066400000000000000000000003561516100555600203340ustar00rootroot00000000000000#!/usr/bin/env node const path = '/tmp/tmp-22366RXowst8OEp2G/github'; const globby = require('globby') const mz = require('mz'); const ini = require('ini') const utils = require('corifeus-utils'); replaceGitSubmodules(path, 'patrikx3');test/scripts/version.js000077500000000000000000000001171516100555600155550ustar00rootroot00000000000000#!/usr/bin/env node console.log(parseInt(process.versions.node.split('.')[0])) tools.iml000066400000000000000000000005171516100555600127300ustar00rootroot00000000000000