ship.zone/npmci
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into 'master'

Browse Source 
Go fully async with shelljs to reduce processor strain

Closes #2

See merge request !2
This commit is contained in:
Phil Kunz 2017-03-08 19:22:39 +00:00
commit bbc2e9002a
17 changed files with 418 additions and 483 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -30,7 +30,6 @@
"dependencies": { "dependencies": {
"@types/lodash": "^4.14.52", "@types/lodash": "^4.14.52",
"@types/node": "^7.0.5", "@types/node": "^7.0.5",
"@types/q": "0.x.x",
"@types/request": "0.x.x", "@types/request": "0.x.x",
"@types/shelljs": "^0.7.0", "@types/shelljs": "^0.7.0",
"@types/through2": "^2.0.32", "@types/through2": "^2.0.32",
@ -40,7 +39,6 @@
"lodash": "^4.17.4", "lodash": "^4.17.4",
"npmextra": "^2.0.3", "npmextra": "^2.0.3",
"projectinfo": "^3.0.1", "projectinfo": "^3.0.1",
"q": "^1.4.1",
"request": "^2.79.0", "request": "^2.79.0",
"shelljs": "^0.7.6", "shelljs": "^0.7.6",
"smartcli": "^2.0.1", "smartcli": "^2.0.1",

8
ts/npmci.bash.ts
View File

@ -19,7 +19,7 @@ checkNvm()
* @param commandArg - The command to execute * @param commandArg - The command to execute
* @param retryArg - The retryArg: 0 to any positive number will retry, -1 will always succeed, -2 will return undefined * @param retryArg - The retryArg: 0 to any positive number will retry, -1 will always succeed, -2 will return undefined
*/ */
export let bash = (commandArg: string, retryArg: number = 2, bareArg: boolean = false): string => { export let bash = async (commandArg: string, retryArg: number = 2, bareArg: boolean = false): Promise<string> => {
let exitCode: number let exitCode: number
let stdOut: string let stdOut: string
let execResult let execResult
@ -43,6 +43,7 @@ export let bash = (commandArg: string, retryArg: number = 2, bareArg: boolean =
// determine how bash reacts to error and success // determine how bash reacts to error and success
if (exitCode !== 0 && i === retryArg) { // something went wrong and retries are exhausted if (exitCode !== 0 && i === retryArg) { // something went wrong and retries are exhausted
if (failOnError) { if (failOnError) {
plugins.beautylog.error('something went wrong and retries are exhausted')
process.exit(1) process.exit(1)
} }
} else if (exitCode === 0) { // everything went fine, or no error wanted } else if (exitCode === 0) { // everything went fine, or no error wanted
@ -55,19 +56,18 @@ export let bash = (commandArg: string, retryArg: number = 2, bareArg: boolean =
} else { } else {
plugins.beautylog.log('ShellExec would be: ' + commandArg) plugins.beautylog.log('ShellExec would be: ' + commandArg)
} }
return stdOut
} }
/** /**
* bashBare allows usage of bash without sourcing any files like nvm * bashBare allows usage of bash without sourcing any files like nvm
*/ */
export let bashBare = (commandArg: string, retryArg: number = 2) => { export let bashBare = (commandArg: string, retryArg: number = 2): Promise<string> => {
return bash(commandArg, retryArg, true) return bash(commandArg, retryArg, true)
} }
/** /**
* bashNoError allows executing stuff without throwing an error * bashNoError allows executing stuff without throwing an error
*/ */
export let bashNoError = (commandArg: string): string => { export let bashNoError = (commandArg: string): Promise<string> => {
return bash(commandArg, -1) return bash(commandArg, -1)
} }

421
ts/npmci.build.docker.ts
View File

@ -1,43 +1,38 @@
import * as plugins from './npmci.plugins' import * as plugins from './npmci.plugins'
import * as paths from './npmci.paths' import * as paths from './npmci.paths'
import * as NpmciEnv from './npmci.env' import * as NpmciEnv from './npmci.env'
import {bashBare} from './npmci.bash' import { bashBare } from './npmci.bash'
/** /**
* builds a cwd of Dockerfiles by triggering a promisechain * builds a cwd of Dockerfiles by triggering a promisechain
*/ */
export let build = function(){ export let build = async () => {
let done = plugins.q.defer() await readDockerfiles()
readDockerfiles() .then(sortDockerfiles)
.then(sortDockerfiles) .then(mapDockerfiles)
.then(mapDockerfiles) .then(buildDockerfiles)
.then(buildDockerfiles) .then(pushDockerfiles)
.then(pushDockerfiles)
.then(() => {
done.resolve()
})
return done.promise
} }
/** /**
* creates instance of class Dockerfile for all Dockerfiles in cwd * creates instance of class Dockerfile for all Dockerfiles in cwd
* @returns Promise<Dockerfile[]> * @returns Promise<Dockerfile[]>
*/ */
export let readDockerfiles = function(): plugins.q.Promise<Dockerfile[]>{ export let readDockerfiles = async (): Promise<Dockerfile[]> => {
let done = plugins.q.defer<Dockerfile[]>() let fileTree = await plugins.smartfile.fs.listFileTree(paths.cwd, './Dockerfile*')
let readDockerfilesArray: Dockerfile[] = []
plugins.gulp.src('./Dockerfile*') // create the Dockerfile array
.pipe(plugins.through2.obj(function(file,enc,cb){ let readDockerfilesArray: Dockerfile[] = []
let myDockerfile = new Dockerfile({ for (let dockerfilePath of fileTree) {
filePath: file.path, let myDockerfile = new Dockerfile({
read: true filePath: dockerfilePath,
}) read: true
readDockerfilesArray.push(myDockerfile) })
cb(null,file) readDockerfilesArray.push(myDockerfile)
},function(){ }
done.resolve(readDockerfilesArray)
})) return readDockerfilesArray
return done.promise
} }
/** /**
@ -45,249 +40,235 @@ export let readDockerfiles = function(): plugins.q.Promise<Dockerfile[]>{
* @param sortableArrayArg an array of instances of class Dockerfile * @param sortableArrayArg an array of instances of class Dockerfile
* @returns Promise<Dockerfile[]> * @returns Promise<Dockerfile[]>
*/ */
export let sortDockerfiles = function(sortableArrayArg: Dockerfile[]): plugins.q.Promise<Dockerfile[]>{ export let sortDockerfiles = (sortableArrayArg: Dockerfile[]): plugins.q.Promise<Dockerfile[]> => {
let done = plugins.q.defer<Dockerfile[]>() let done = plugins.q.defer<Dockerfile[]>()
let sortedArray: Dockerfile[] = [] let sortedArray: Dockerfile[] = []
let cleanTagsOriginal = cleanTagsArrayFunction(sortableArrayArg,sortedArray) let cleanTagsOriginal = cleanTagsArrayFunction(sortableArrayArg, sortedArray)
let sorterFunctionCounter: number = 0 let sorterFunctionCounter: number = 0
let sorterFunction = function(){ let sorterFunction = function () {
sortableArrayArg.forEach((dockerfileArg) => { sortableArrayArg.forEach((dockerfileArg) => {
let cleanTags = cleanTagsArrayFunction(sortableArrayArg,sortedArray) let cleanTags = cleanTagsArrayFunction(sortableArrayArg, sortedArray)
if (cleanTags.indexOf(dockerfileArg.baseImage) === -1 && sortedArray.indexOf(dockerfileArg) === -1) { if (cleanTags.indexOf(dockerfileArg.baseImage) === -1 && sortedArray.indexOf(dockerfileArg) === -1) {
sortedArray.push(dockerfileArg) sortedArray.push(dockerfileArg)
}; };
if (cleanTagsOriginal.indexOf(dockerfileArg.baseImage) !== -1) { if (cleanTagsOriginal.indexOf(dockerfileArg.baseImage) !== -1) {
dockerfileArg.localBaseImageDependent = true dockerfileArg.localBaseImageDependent = true
}; };
}) })
if (sortableArrayArg.length === sortedArray.length) { if (sortableArrayArg.length === sortedArray.length) {
done.resolve(sortedArray) done.resolve(sortedArray)
} else if (sorterFunctionCounter < 10) { } else if (sorterFunctionCounter < 10) {
sorterFunctionCounter++ sorterFunctionCounter++
sorterFunction() sorterFunction()
}; };
} }
sorterFunction() sorterFunction()
return done.promise return done.promise
} }
/** /**
* maps local Dockerfiles dependencies to the correspoding Dockerfile class instances * maps local Dockerfiles dependencies to the correspoding Dockerfile class instances
*/ */
export let mapDockerfiles = function(sortedArray: Dockerfile[]): plugins.q.Promise<Dockerfile[]>{ export let mapDockerfiles = async (sortedArray: Dockerfile[]): plugins.q.Promise<Dockerfile[]> => {
let done = plugins.q.defer<Dockerfile[]>() sortedArray.forEach((dockerfileArg) => {
sortedArray.forEach((dockerfileArg) => { if (dockerfileArg.localBaseImageDependent) {
if (dockerfileArg.localBaseImageDependent) { sortedArray.forEach((dockfile2: Dockerfile) => {
sortedArray.forEach((dockfile2: Dockerfile) => { if (dockfile2.cleanTag === dockerfileArg.baseImage) {
if (dockfile2.cleanTag === dockerfileArg.baseImage) { dockerfileArg.localBaseDockerfile = dockfile2
dockerfileArg.localBaseDockerfile = dockfile2 }
} })
}) };
}; })
}) return sortedArray
done.resolve(sortedArray)
return done.promise
} }
/** /**
* builds the correspoding real docker image for each Dockerfile class instance * builds the correspoding real docker image for each Dockerfile class instance
*/ */
export let buildDockerfiles = (sortedArrayArg: Dockerfile[]) => { export let buildDockerfiles = async (sortedArrayArg: Dockerfile[]) => {
let done = plugins.q.defer() sortedArrayArg.forEach(async function (dockerfileArg) {
sortedArrayArg.forEach(function(dockerfileArg){ await dockerfileArg.build()
dockerfileArg.build() })
}) return sortedArrayArg
done.resolve(sortedArrayArg)
return done.promise
} }
/** /**
* pushes the real Dockerfile images to a Docker registry * pushes the real Dockerfile images to a Docker registry
*/ */
export let pushDockerfiles = function(sortedArrayArg: Dockerfile[]){ export let pushDockerfiles = async (sortedArrayArg: Dockerfile[]) => {
let done = plugins.q.defer() sortedArrayArg.forEach(async (dockerfileArg) => {
sortedArrayArg.forEach(function(dockerfileArg){ await dockerfileArg.push(NpmciEnv.buildStage)
dockerfileArg.push(NpmciEnv.buildStage) })
}) return sortedArrayArg
done.resolve(sortedArrayArg)
return done.promise
} }
/** /**
* pulls corresponding real Docker images for instances of Dockerfile from a registry. * pulls corresponding real Docker images for instances of Dockerfile from a registry.
* This is needed if building, testing, and publishing of Docker images is carried out in seperate CI stages. * This is needed if building, testing, and publishing of Docker images is carried out in seperate CI stages.
*/ */
export let pullDockerfileImages = (sortableArrayArg: Dockerfile[],registryArg = 'registry.gitlab.com') => { export let pullDockerfileImages = async (sortableArrayArg: Dockerfile[], registryArg = 'registry.gitlab.com') => {
let done = plugins.q.defer() sortableArrayArg.forEach(async (dockerfileArg) => {
sortableArrayArg.forEach((dockerfileArg) => { await dockerfileArg.pull(registryArg)
dockerfileArg.pull(registryArg) })
}) return sortableArrayArg
done.resolve(sortableArrayArg)
return done.promise
} }
/** /**
* tests all Dockerfiles in by calling class Dockerfile.test(); * tests all Dockerfiles in by calling class Dockerfile.test();
* @param sortedArrayArg Dockerfile[] that contains all Dockerfiles in cwd * @param sortedArrayArg Dockerfile[] that contains all Dockerfiles in cwd
*/ */
export let testDockerfiles = (sortedArrayArg: Dockerfile[]) => { export let testDockerfiles = async (sortedArrayArg: Dockerfile[]) => {
let done = plugins.q.defer() sortedArrayArg.forEach(async (dockerfileArg) => {
sortedArrayArg.forEach(function(dockerfileArg){ await dockerfileArg.test()
dockerfileArg.test() })
}) return sortedArrayArg
done.resolve(sortedArrayArg)
return done.promise
} }
/** /**
* class Dockerfile represents a Dockerfile on disk in npmci * class Dockerfile represents a Dockerfile on disk in npmci
*/ */
export class Dockerfile { export class Dockerfile {
filePath: string filePath: string
repo: string repo: string
version: string version: string
cleanTag: string cleanTag: string
buildTag: string buildTag: string
testTag: string testTag: string
releaseTag: string releaseTag: string
containerName: string containerName: string
content: string content: string
baseImage: string baseImage: string
localBaseImageDependent: boolean localBaseImageDependent: boolean
localBaseDockerfile: Dockerfile localBaseDockerfile: Dockerfile
constructor(options: {filePath?: string,fileContents?: string|Buffer,read?: boolean}) { constructor(options: { filePath?: string, fileContents?: string | Buffer, read?: boolean }) {
this.filePath = options.filePath this.filePath = options.filePath
this.repo = NpmciEnv.repo.user + '/' + NpmciEnv.repo.repo this.repo = NpmciEnv.repo.user + '/' + NpmciEnv.repo.repo
this.version = dockerFileVersion(plugins.path.parse(options.filePath).base) this.version = dockerFileVersion(plugins.path.parse(options.filePath).base)
this.cleanTag = this.repo + ':' + this.version this.cleanTag = this.repo + ':' + this.version
this.buildTag = this.cleanTag this.buildTag = this.cleanTag
this.testTag = dockerTag('registry.gitlab.com',this.repo,this.version,'test') this.testTag = dockerTag('registry.gitlab.com', this.repo, this.version, 'test')
this.releaseTag = dockerTag(NpmciEnv.dockerRegistry,this.repo,this.version) this.releaseTag = dockerTag(NpmciEnv.dockerRegistry, this.repo, this.version)
this.containerName = 'dockerfile-' + this.version this.containerName = 'dockerfile-' + this.version
if (options.filePath && options.read) { if (options.filePath && options.read) {
this.content = plugins.smartfile.fs.toStringSync(plugins.path.resolve(options.filePath)) this.content = plugins.smartfile.fs.toStringSync(plugins.path.resolve(options.filePath))
};
this.baseImage = dockerBaseImage(this.content)
this.localBaseImageDependent = false
}; };
this.baseImage = dockerBaseImage(this.content)
this.localBaseImageDependent = false
};
/** /**
* builds the Dockerfile * builds the Dockerfile
*/ */
build() { async build() {
let done = plugins.q.defer() plugins.beautylog.info('now building Dockerfile for ' + this.cleanTag)
plugins.beautylog.info('now building Dockerfile for ' + this.cleanTag) await bashBare('docker build -t ' + this.buildTag + ' -f ' + this.filePath + ' .')
bashBare('docker build -t ' + this.buildTag + ' -f ' + this.filePath + ' .') NpmciEnv.dockerFilesBuilt.push(this)
NpmciEnv.dockerFilesBuilt.push(this) };
done.resolve()
return done.promise
};
/** /**
* pushes the Dockerfile to a registry * pushes the Dockerfile to a registry
*/ */
push(stageArg) { async push(stageArg) {
let done = plugins.q.defer() let pushTag
let pushTag switch (stageArg) {
switch (stageArg) { case 'release':
case 'release': pushTag = this.releaseTag
pushTag = this.releaseTag break
break case 'test':
case 'test': default:
default: pushTag = this.testTag
pushTag = this.testTag break
break
}
bashBare('docker tag ' + this.buildTag + ' ' + pushTag)
bashBare('docker push ' + pushTag)
done.resolve()
return done.promise
};
/**
* pulls the Dockerfile from a registry
*/
pull(registryArg: string) {
let pullTag = this.testTag
bashBare('docker pull ' + pullTag)
bashBare('docker tag ' + pullTag + ' ' + this.buildTag)
};
/**
* tests the Dockerfile;
*/
test() {
let testFile: string = plugins.path.join(paths.NpmciTestDir,'test_' + this.version + '.sh')
let testFileExists: boolean = plugins.smartfile.fs.fileExistsSync(testFile)
if (testFileExists) {
bashBare('docker run --name npmci_test_container ' + this.buildTag + ' mkdir /npmci_test')
bashBare('docker cp ' + testFile + ' npmci_test_container:/npmci_test/test.sh')
bashBare('docker commit npmci_test_container npmci_test_image')
bashBare('docker run npmci_test_image sh /npmci_test/test.sh')
bashBare('docker rm npmci_test_container')
bashBare('docker rmi --force npmci_test_image')
} else {
plugins.beautylog.warn('skipping tests for ' + this.cleanTag + ' because no testfile was found!')
}
};
/**
* gets the id of a Dockerfile
*/
getId() {
let containerId = bashBare('docker inspect --type=image --format=\"{{.Id}}\" ' + this.buildTag)
return containerId
};
}
/**
*
*/
export let dockerFileVersion = function(dockerfileNameArg: string): string{
let versionString: string
let versionRegex = /Dockerfile_([a-zA-Z0-9\.]*)$/
let regexResultArray = versionRegex.exec(dockerfileNameArg)
if (regexResultArray && regexResultArray.length === 2) {
versionString = regexResultArray[1]
} else {
versionString = 'latest'
} }
return versionString await bashBare('docker tag ' + this.buildTag + ' ' + pushTag)
await bashBare('docker push ' + pushTag)
};
/**
* pulls the Dockerfile from a registry
*/
async pull(registryArg: string) {
let pullTag = this.testTag
await bashBare('docker pull ' + pullTag)
await bashBare('docker tag ' + pullTag + ' ' + this.buildTag)
};
/**
* tests the Dockerfile;
*/
async test() {
let testFile: string = plugins.path.join(paths.NpmciTestDir, 'test_' + this.version + '.sh')
let testFileExists: boolean = plugins.smartfile.fs.fileExistsSync(testFile)
if (testFileExists) {
// run tests
await bashBare('docker run --name npmci_test_container ' + this.buildTag + ' mkdir /npmci_test')
await bashBare('docker cp ' + testFile + ' npmci_test_container:/npmci_test/test.sh')
await bashBare('docker commit npmci_test_container npmci_test_image')
await bashBare('docker run npmci_test_image sh /npmci_test/test.sh')
await bashBare('docker rm npmci_test_container')
await bashBare('docker rmi --force npmci_test_image')
} else {
plugins.beautylog.warn('skipping tests for ' + this.cleanTag + ' because no testfile was found!')
}
};
/**
* gets the id of a Dockerfile
*/
async getId() {
let containerId = await bashBare('docker inspect --type=image --format=\"{{.Id}}\" ' + this.buildTag)
return containerId
};
}
/**
* returns a version for a docker file
* @execution SYNC
*/
export let dockerFileVersion = (dockerfileNameArg: string): string => {
let versionString: string
let versionRegex = /Dockerfile_([a-zA-Z0-9\.]*)$/
let regexResultArray = versionRegex.exec(dockerfileNameArg)
if (regexResultArray && regexResultArray.length === 2) {
versionString = regexResultArray[1]
} else {
versionString = 'latest'
}
return versionString
} }
/** /**
* *
*/ */
export let dockerBaseImage = function(dockerfileContentArg: string){ export let dockerBaseImage = function (dockerfileContentArg: string) {
let baseImageRegex = /FROM\s([a-zA-z0-9\/\-\:]*)\n?/ let baseImageRegex = /FROM\s([a-zA-z0-9\/\-\:]*)\n?/
let regexResultArray = baseImageRegex.exec(dockerfileContentArg) let regexResultArray = baseImageRegex.exec(dockerfileContentArg)
return regexResultArray[1] return regexResultArray[1]
} }
/** /**
* *
*/ */
export let dockerTag = function(registryArg: string,repoArg: string,versionArg: string,suffixArg?: string): string{ export let dockerTag = function (registryArg: string, repoArg: string, versionArg: string, suffixArg?: string): string {
let tagString: string let tagString: string
let registry = registryArg let registry = registryArg
let repo = repoArg let repo = repoArg
let version = versionArg let version = versionArg
if (suffixArg) { if (suffixArg) {
version = versionArg + '_' + suffixArg version = versionArg + '_' + suffixArg
}; };
tagString = registry + '/' + repo + ':' + version tagString = registry + '/' + repo + ':' + version
return tagString return tagString
} }
/** /**
* *
*/ */
export let cleanTagsArrayFunction = function(dockerfileArrayArg: Dockerfile[],trackingArrayArg: Dockerfile[]): string[]{ export let cleanTagsArrayFunction = function (dockerfileArrayArg: Dockerfile[], trackingArrayArg: Dockerfile[]): string[] {
let cleanTagsArray: string[] = [] let cleanTagsArray: string[] = []
dockerfileArrayArg.forEach(function(dockerfileArg){ dockerfileArrayArg.forEach(function (dockerfileArg) {
if (trackingArrayArg.indexOf(dockerfileArg) === -1) { if (trackingArrayArg.indexOf(dockerfileArg) === -1) {
cleanTagsArray.push(dockerfileArg.cleanTag) cleanTagsArray.push(dockerfileArg.cleanTag)
} }
}) })
return cleanTagsArray return cleanTagsArray
} }

21
ts/npmci.build.ts
View File

@ -1,22 +1,23 @@
import * as plugins from './npmci.plugins' import * as plugins from './npmci.plugins'
import {bash} from './npmci.bash' import { bash } from './npmci.bash'
import * as env from './npmci.env' import * as env from './npmci.env'
import * as buildDocker from './npmci.build.docker' import * as buildDocker from './npmci.build.docker'
/** /**
* defines possible build services * defines possible build services
*/ */
export type TBuildService = 'docker'; export type TBuildService = 'docker'
/** /**
* builds for a specific service * builds for a specific service
*/ */
export let build = function(commandArg): plugins.q.Promise<any> { export let build = async (commandArg): Promise<void> => {
switch (commandArg) { switch (commandArg) {
case 'docker': case 'docker':
return buildDocker.build() await buildDocker.build()
default: break
plugins.beautylog.log('build target ' + commandArg + ' not recognised!') default:
}; plugins.beautylog.log('build target ' + commandArg + ' not recognised!')
return };
return
} }

8
ts/npmci.clean.ts
View File

@ -4,9 +4,7 @@ import * as paths from './npmci.paths'
/** /**
* cleans npmci config files * cleans npmci config files
*/ */
export let clean = () => { export let clean = async (): Promise<void> => {
let done = plugins.q.defer() plugins.smartfile.fs.removeSync(paths.NpmciPackageConfig)
plugins.smartfile.fs.removeSync(paths.NpmciPackageConfig) return
done.resolve()
return done.promise
} }

22
ts/npmci.command.ts
View File

@ -1,15 +1,13 @@
import * as plugins from './npmci.plugins' import * as plugins from './npmci.plugins'
import {bash} from './npmci.bash' import { bash } from './npmci.bash'
export let command = () => { export let command = async () => {
let done = plugins.q.defer() let wrappedCommand: string = ''
let wrappedCommand: string = '' let argvArray = process.argv
let argvArray = process.argv for (let i = 3; i < argvArray.length; i++) {
for (let i = 3; i < argvArray.length; i++) { wrappedCommand = wrappedCommand + argvArray[i]
wrappedCommand = wrappedCommand + argvArray[i] if (i + 1 !== argvArray.length) { wrappedCommand = wrappedCommand + ' ' }
if (i + 1 !== argvArray.length) { wrappedCommand = wrappedCommand + ' ' } }
} await bash(wrappedCommand)
bash(wrappedCommand) return
done.resolve()
return done.promise
} }

18
ts/npmci.config.ts
View File

@ -4,16 +4,14 @@ import * as plugins from './npmci.plugins'
import * as paths from './npmci.paths' import * as paths from './npmci.paths'
export interface INpmciOptions { export interface INpmciOptions {
globalNpmTools: string[] globalNpmTools: string[]
} }
export let getConfig = () => { export let getConfig = async (): Promise<INpmciOptions> => {
let done = q.defer<INpmciOptions>() let npmciNpmextra = new plugins.npmextra.Npmextra(paths.cwd)
let npmciNpmextra = new plugins.npmextra.Npmextra(paths.cwd) let defaultConfig: INpmciOptions = {
let defaultConfig: INpmciOptions = { globalNpmTools: []
globalNpmTools: [] }
} let npmciConfig = npmciNpmextra.dataFor<INpmciOptions>('npmci', defaultConfig)
let npmciConfig = npmciNpmextra.dataFor<INpmciOptions>('npmci', defaultConfig) return npmciConfig
done.resolve(npmciConfig)
return done.promise
} }

65
ts/npmci.env.ts
View File

@ -1,7 +1,7 @@
import * as plugins from './npmci.plugins' import * as plugins from './npmci.plugins'
import * as paths from './npmci.paths' import * as paths from './npmci.paths'
import {GitRepo} from 'smartstring' import { GitRepo } from 'smartstring'
import {Dockerfile} from './npmci.build.docker' import { Dockerfile } from './npmci.build.docker'
export let repo: GitRepo export let repo: GitRepo
if (process.env.CI_BUILD_REPO) repo = new GitRepo(process.env.CI_BUILD_REPO) if (process.env.CI_BUILD_REPO) repo = new GitRepo(process.env.CI_BUILD_REPO)
@ -11,48 +11,47 @@ export let buildStage: string = process.env.CI_BUILD_STAGE
// handling config between commands // handling config between commands
export let dockerRegistry: string // will be set by npmci.prepare export let dockerRegistry: string // will be set by npmci.prepare
export let setDockerRegistry = (dockerRegistryArg: string) => { export let setDockerRegistry = (dockerRegistryArg: string) => {
dockerRegistry = dockerRegistryArg dockerRegistry = dockerRegistryArg
} }
export let dockerFilesBuilt: Dockerfile[] = [] export let dockerFilesBuilt: Dockerfile[] = []
export let dockerFiles: Dockerfile[] = [] export let dockerFiles: Dockerfile[] = []
export let config = { export let config = {
dockerRegistry: undefined, // this will be set later on store dockerRegistry: undefined, // this will be set later on store
dockerFilesBuilt: dockerFilesBuilt, dockerFilesBuilt: dockerFilesBuilt,
dockerFiles: dockerFiles, dockerFiles: dockerFiles,
project: undefined project: undefined
} }
export let configStore = () => { export let configStore = () => {
config.dockerRegistry = dockerRegistry config.dockerRegistry = dockerRegistry
plugins.smartfile.memory.toFsSync( plugins.smartfile.memory.toFsSync(
JSON.stringify(config), JSON.stringify(config),
paths.NpmciPackageConfig paths.NpmciPackageConfig
) )
} }
let configLoad = () => { let configLoad = () => {
// internal config to transfer information in between npmci shell calls // internal config to transfer information in between npmci shell calls
try { try {
plugins.lodash.assign(config,plugins.smartfile.fs.toObjectSync(paths.NpmciPackageConfig,'json')) plugins.lodash.assign(config, plugins.smartfile.fs.toObjectSync(paths.NpmciPackageConfig, 'json'))
} } catch (err) {
catch (err) { configStore()
configStore() plugins.beautylog.log('config initialized!')
plugins.beautylog.log('config initialized!') }
}
// project config // project config
try { try {
if (!config.project) { if (!config.project) {
config.project = plugins.smartfile.fs.toObjectSync(paths.NpmciProjectDir,'npmci.json') config.project = plugins.smartfile.fs.toObjectSync(paths.NpmciProjectDir, 'npmci.json')
plugins.beautylog.ok('project config found!') plugins.beautylog.ok('project config found!')
}; };
} }
catch (err) { catch (err) {
config.project = {} config.project = {}
plugins.beautylog.log('no project config found, so proceeding with default behaviour!') plugins.beautylog.log('no project config found, so proceeding with default behaviour!')
} }
config.dockerRegistry ? dockerRegistry = config.dockerRegistry : void(0) config.dockerRegistry ? dockerRegistry = config.dockerRegistry : void (0)
config.dockerFilesBuilt ? dockerFilesBuilt = config.dockerFilesBuilt : void(0) config.dockerFilesBuilt ? dockerFilesBuilt = config.dockerFilesBuilt : void (0)
} }
configLoad() configLoad()

22
ts/npmci.install.ts
View File

@ -2,8 +2,12 @@ import * as plugins from './npmci.plugins'
import * as configModule from './npmci.config' import * as configModule from './npmci.config'
import { bash, bashNoError } from './npmci.bash' import { bash, bashNoError } from './npmci.bash'
import { nvmAvailable } from './npmci.bash' import { nvmAvailable } from './npmci.bash'
export let install = (versionArg) => {
let done = plugins.q.defer() /**
* Install a specific version of node
* @param versionArg
*/
export let install = async (versionArg) => {
plugins.beautylog.log(`now installing node version ${versionArg}`) plugins.beautylog.log(`now installing node version ${versionArg}`)
let version: string let version: string
if (versionArg === 'stable') { if (versionArg === 'stable') {
@ -16,30 +20,28 @@ export let install = (versionArg) => {
version = versionArg version = versionArg
}; };
if (nvmAvailable) { if (nvmAvailable) {
bash(`nvm install ${version} && nvm alias default ${version}`) await bash(`nvm install ${version} && nvm alias default ${version}`)
plugins.beautylog.success(`Node version ${version} successfully installed!`) plugins.beautylog.success(`Node version ${version} successfully installed!`)
} else { } else {
plugins.beautylog.warn('Nvm not in path so staying at installed node version!') plugins.beautylog.warn('Nvm not in path so staying at installed node version!')
}; };
bash('node -v') await bash('node -v')
bash('npm -v') await bash('npm -v')
// lets look for further config // lets look for further config
configModule.getConfig() configModule.getConfig()
.then(configArg => { .then(async configArg => {
plugins.beautylog.log('Now checking for needed global npm tools...') plugins.beautylog.log('Now checking for needed global npm tools...')
for (let npmTool of configArg.globalNpmTools) { for (let npmTool of configArg.globalNpmTools) {
plugins.beautylog.info(`Checking for global "${npmTool}"`) plugins.beautylog.info(`Checking for global "${npmTool}"`)
let whichOutput = bashNoError(`which ${npmTool}`) let whichOutput: string = await bashNoError(`which ${npmTool}`)
let toolAvailable: boolean = !((/not\sfound/.test(whichOutput)) || whichOutput === '') let toolAvailable: boolean = !((/not\sfound/.test(whichOutput)) || whichOutput === '')
if (toolAvailable) { if (toolAvailable) {
plugins.beautylog.log(`Tool ${npmTool} is available`) plugins.beautylog.log(`Tool ${npmTool} is available`)
} else { } else {
plugins.beautylog.info(`globally installing ${npmTool} from npm`) plugins.beautylog.info(`globally installing ${npmTool} from npm`)
bash(`npm install ${npmTool} -q -g`) await bash(`npm install ${npmTool} -q -g`)
} }
} }
plugins.beautylog.success('all global npm tools specified in npmextra.json are now available!') plugins.beautylog.success('all global npm tools specified in npmextra.json are now available!')
done.resolve()
}) })
return done.promise
} }

2
ts/npmci.plugins.ts
View File

@ -5,7 +5,7 @@ export import lodash = require('lodash')
export import npmextra = require('npmextra') export import npmextra = require('npmextra')
export import path = require('path') export import path = require('path')
export import projectinfo = require('projectinfo') export import projectinfo = require('projectinfo')
export import q = require('q') export import q = require('smartq')
export let request = require('request') export let request = require('request')
export import shelljs = require('shelljs') export import shelljs = require('shelljs')
export import smartcli = require('smartcli') export import smartcli = require('smartcli')

103
ts/npmci.prepare.ts
View File

@ -1,5 +1,5 @@
import * as plugins from './npmci.plugins' import * as plugins from './npmci.plugins'
import {bash} from './npmci.bash' import { bash } from './npmci.bash'
import * as env from './npmci.env' import * as env from './npmci.env'
import * as sshModule from './npmci.ssh' import * as sshModule from './npmci.ssh'
@ -9,85 +9,74 @@ import * as sshModule from './npmci.ssh'
/** /**
* defines possible prepare services * defines possible prepare services
*/ */
export type TPrepService = 'npm' | 'docker' | 'docker-gitlab' | 'ssh'; export type TPrepService = 'npm' | 'docker' | 'docker-gitlab' | 'ssh'
/** /**
* authenticates npm with token from env var * authenticates npm with token from env var
*/ */
let npm = function(){ let npm = async () => {
let done = plugins.q.defer() let npmrcPrefix: string = '//registry.npmjs.org/:_authToken='
let npmToken: string = process.env.NPMCI_TOKEN_NPM
let npmrcPrefix: string = '//registry.npmjs.org/:_authToken=' let npmrcFileString: string = npmrcPrefix + npmToken
let npmToken: string = process.env.NPMCI_TOKEN_NPM if (npmToken) {
let npmrcFileString = npmrcPrefix + npmToken plugins.beautylog.info('found access token')
} else {
if (npmToken) { plugins.beautylog.error('no access token found! Exiting!')
plugins.beautylog.info('found access token') process.exit(1)
} else { }
plugins.beautylog.error('no access token found! Exiting!') plugins.smartfile.memory.toFsSync(npmrcFileString, '/root/.npmrc')
process.exit(1) return
}
plugins.smartfile.memory.toFsSync(npmrcFileString,'/root/.npmrc')
done.resolve()
return done.promise
} }
/** /**
* logs in docker * logs in docker
*/ */
let docker = function(){ let docker = async () => {
let done = plugins.q.defer() env.setDockerRegistry('docker.io')
env.setDockerRegistry('docker.io') let dockerRegex = /^([a-zA-Z0-9\.]*)\|([a-zA-Z0-9\.]*)/
let dockerRegex = /^([a-zA-Z0-9\.]*)\|([a-zA-Z0-9\.]*)/ if (!process.env.NPMCI_LOGIN_DOCKER) {
if (!process.env.NPMCI_LOGIN_DOCKER) { plugins.beautylog.error('You have to specify Login Data to the Docker Registry')
plugins.beautylog.error('You have to specify Login Data to the Docker Registry') process.exit(1)
process.exit(1) }
} plugins.shelljs.exec('docker login -u gitlab-ci-token -p ' + process.env.CI_BUILD_TOKEN + ' ' + 'registry.gitlab.com') // Always also login to GitLab Registry
plugins.shelljs.exec('docker login -u gitlab-ci-token -p ' + process.env.CI_BUILD_TOKEN + ' ' + 'registry.gitlab.com') // Always also login to GitLab Registry let dockerRegexResultArray = dockerRegex.exec(process.env.NPMCI_LOGIN_DOCKER)
let dockerRegexResultArray = dockerRegex.exec(process.env.NPMCI_LOGIN_DOCKER) let username = dockerRegexResultArray[1]
let username = dockerRegexResultArray[1] let password = dockerRegexResultArray[2]
let password = dockerRegexResultArray[2] plugins.shelljs.exec('docker login -u ' + username + ' -p ' + password)
plugins.shelljs.exec('docker login -u ' + username + ' -p ' + password) return
done.resolve()
return done.promise
} }
/** /**
* prepare docker for gitlab registry * prepare docker for gitlab registry
*/ */
let dockerGitlab = function(){ let dockerGitlab = async () => {
let done = plugins.q.defer() env.setDockerRegistry('registry.gitlab.com')
env.setDockerRegistry('registry.gitlab.com') plugins.shelljs.exec('docker login -u gitlab-ci-token -p ' + process.env.CI_BUILD_TOKEN + ' ' + 'registry.gitlab.com')
plugins.shelljs.exec('docker login -u gitlab-ci-token -p ' + process.env.CI_BUILD_TOKEN + ' ' + 'registry.gitlab.com') return
done.resolve()
return done.promise
} }
/** /**
* prepare ssh * prepare ssh
*/ */
let ssh = function(){ let ssh = async () => {
let done = plugins.q.defer() await sshModule.ssh()
sshModule.ssh()
.then(done.resolve)
return done.promise
} }
/** /**
* the main exported prepare function * the main exported prepare function
* @param servieArg describes the service to prepare * @param servieArg describes the service to prepare
*/ */
export let prepare = function(serviceArg: TPrepService){ export let prepare = async (serviceArg: TPrepService) => {
switch (serviceArg) { switch (serviceArg) {
case 'npm': case 'npm':
return npm() return await npm()
case 'docker': case 'docker':
return docker() return await docker()
case 'docker-gitlab': case 'docker-gitlab':
return dockerGitlab() return await dockerGitlab()
case 'ssh': case 'ssh':
return ssh() return await ssh()
default: default:
break break
} }
} }

49
ts/npmci.publish.ts
View File

@ -1,49 +1,46 @@
import * as plugins from './npmci.plugins' import * as plugins from './npmci.plugins'
import {prepare} from './npmci.prepare' import { prepare } from './npmci.prepare'
import {bash} from './npmci.bash' import { bash } from './npmci.bash'
import * as NpmciEnv from './npmci.env' import * as NpmciEnv from './npmci.env'
import * as NpmciBuildDocker from './npmci.build.docker' import * as NpmciBuildDocker from './npmci.build.docker'
/** /**
* type of supported services * type of supported services
*/ */
export type TPubService = 'npm' | 'docker'; export type TPubService = 'npm' | 'docker'
/** /**
* the main exported publish function. * the main exported publish function.
* @param pubServiceArg references targeted service to publish to * @param pubServiceArg references targeted service to publish to
*/ */
export let publish = (pubServiceArg: TPubService = 'npm') => { export let publish = async (pubServiceArg: TPubService = 'npm') => {
switch (pubServiceArg) { switch (pubServiceArg) {
case 'npm': case 'npm':
return publishNpm() return await publishNpm()
case 'docker': case 'docker':
return publishDocker() return await publishDocker()
} }
} }
/** /**
* tries to publish current cwd to NPM registry * tries to publish current cwd to NPM registry
*/ */
let publishNpm = function(){ let publishNpm = async () => {
let done = plugins.q.defer() await prepare('npm')
prepare('npm') .then(async function () {
.then(function(){ await bash('npm publish')
bash('npm publish') plugins.beautylog.ok('Done!')
plugins.beautylog.ok('Done!') })
done.resolve()
})
return done.promise
} }
/** /**
* tries to pubish current cwd to Docker registry * tries to pubish current cwd to Docker registry
*/ */
let publishDocker = function(){ let publishDocker = async () => {
let done = plugins.q.defer() return await NpmciBuildDocker.readDockerfiles()
NpmciBuildDocker.readDockerfiles() .then(NpmciBuildDocker.pullDockerfileImages)
.then(NpmciBuildDocker.pullDockerfileImages) .then(NpmciBuildDocker.pushDockerfiles)
.then(NpmciBuildDocker.pushDockerfiles) .then(dockerfileArray => {
.then(done.resolve) return dockerfileArray
return done.promise })
} }

2
ts/npmci.servezone.ts
View File

@ -25,7 +25,7 @@ let smartsocketClientConstructorOptions = {
/** /**
* the main run function to submit a service to a servezone * the main run function to submit a service to a servezone
*/ */
export let run = (configArg) => { export let run = async (configArg) => {
new plugins.smartsocket.SmartsocketClient( new plugins.smartsocket.SmartsocketClient(
smartsocketClientConstructorOptions smartsocketClientConstructorOptions
) )

56
ts/npmci.ssh.ts
View File

@ -6,45 +6,43 @@ let sshInstance: plugins.smartssh.SshInstance
/** /**
* checks for ENV vars in form of NPMCI_SSHKEY_* and deploys any found ones * checks for ENV vars in form of NPMCI_SSHKEY_* and deploys any found ones
*/ */
export let ssh = () => { export let ssh = async () => {
let done = plugins.q.defer() sshInstance = new plugins.smartssh.SshInstance() // init ssh instance
sshInstance = new plugins.smartssh.SshInstance() // init ssh instance plugins.smartparam.forEachMinimatch(process.env, 'NPMCI_SSHKEY_*', evaluateSshEnv)
plugins.smartparam.forEachMinimatch(process.env,'NPMCI_SSHKEY_*',evaluateSshEnv) if (!process.env.NPMTS_TEST) {
if (!process.env.NPMTS_TEST) { sshInstance.writeToDisk()
sshInstance.writeToDisk() } else {
} else { plugins.beautylog.log('In test mode, so not storing SSH keys to disk!')
plugins.beautylog.log('In test mode, so not storing SSH keys to disk!') };
};
done.resolve()
return done.promise
} }
/** /**
* gets called for each found SSH ENV Var and deploys it * gets called for each found SSH ENV Var and deploys it
*/ */
let evaluateSshEnv = (sshkeyEnvVarArg) => { let evaluateSshEnv = async (sshkeyEnvVarArg) => {
let resultArray = sshRegex.exec(sshkeyEnvVarArg) let resultArray = sshRegex.exec(sshkeyEnvVarArg)
let sshKey = new plugins.smartssh.SshKey() let sshKey = new plugins.smartssh.SshKey()
plugins.beautylog.info('Found SSH identity for ' + resultArray[1]) plugins.beautylog.info('Found SSH identity for ' + resultArray[1])
if (notUndefined(resultArray[1])) { if (notUndefined(resultArray[1])) {
plugins.beautylog.log('---> host defined!') plugins.beautylog.log('---> host defined!')
sshKey.host = resultArray[1] sshKey.host = resultArray[1]
} }
if (notUndefined(resultArray[2])) { if (notUndefined(resultArray[2])) {
plugins.beautylog.log('---> privKey defined!') plugins.beautylog.log('---> privKey defined!')
sshKey.privKeyBase64 = resultArray[2] sshKey.privKeyBase64 = resultArray[2]
}; };
if (notUndefined(resultArray[3])) { if (notUndefined(resultArray[3])) {
'---> pubKey defined!' '---> pubKey defined!'
sshKey.pubKeyBase64 = resultArray[3] sshKey.pubKeyBase64 = resultArray[3]
}; };
sshInstance.addKey(sshKey) sshInstance.addKey(sshKey)
return
} }
/** /**
* checks if not undefined * checks if not undefined
*/ */
let notUndefined = (stringArg: string) => { let notUndefined = (stringArg: string) => {
return (stringArg && stringArg !== 'undefined' && stringArg !== '##') return (stringArg && stringArg !== 'undefined' && stringArg !== '##')
} }

57
ts/npmci.test.ts
View File

@ -1,49 +1,32 @@
import * as plugins from './npmci.plugins' import * as plugins from './npmci.plugins'
import {bash} from './npmci.bash' import { bash } from './npmci.bash'
import {install} from './npmci.install' import { install } from './npmci.install'
import * as env from './npmci.env' import * as env from './npmci.env'
import * as NpmciBuildDocker from './npmci.build.docker' import * as NpmciBuildDocker from './npmci.build.docker'
export let test = (versionArg) => { export let test = async (versionArg): Promise<void> => {
let done = plugins.q.defer() if (versionArg === 'docker') {
if (versionArg === 'docker') { await testDocker()
testDocker() } else {
.then(() => { await install(versionArg)
done.resolve() .then(npmDependencies)
}) .then(npmTest)
} else { }
install(versionArg)
.then(npmDependencies)
.then(npmTest)
.then(() => {
done.resolve()
})
}
return done.promise
} }
let npmDependencies = function(){ let npmDependencies = async ():Promise <void> => {
let done = plugins.q.defer() plugins.beautylog.info('now installing dependencies:')
plugins.beautylog.info('now installing dependencies:') await bash('npm install')
bash('npm install')
done.resolve()
return done.promise
} }
let npmTest = () => { let npmTest = async (): Promise<void> => {
let done = plugins.q.defer() plugins.beautylog.info('now starting tests:')
plugins.beautylog.info('now starting tests:') await bash('npm test')
bash('npm test')
done.resolve()
return done.promise
} }
let testDocker = function(){ let testDocker = async (): Promise<NpmciBuildDocker.Dockerfile[]> => {
let done = plugins.q.defer() return await NpmciBuildDocker.readDockerfiles()
NpmciBuildDocker.readDockerfiles() .then(NpmciBuildDocker.pullDockerfileImages)
.then(NpmciBuildDocker.pullDockerfileImages) .then(NpmciBuildDocker.testDockerfiles)
.then(NpmciBuildDocker.testDockerfiles)
.then(done.resolve)
return done.promise
} }

39
ts/npmci.trigger.ts
View File

@ -4,27 +4,24 @@ import { bash } from './npmci.bash'
let triggerValueRegex = /^([a-zA-Z0-9\.]*)\|([a-zA-Z0-9\.]*)\|([a-zA-Z0-9\.]*)\|([a-zA-Z0-9\.]*)\|?([a-zA-Z0-9\.\-\/]*)/ let triggerValueRegex = /^([a-zA-Z0-9\.]*)\|([a-zA-Z0-9\.]*)\|([a-zA-Z0-9\.]*)\|([a-zA-Z0-9\.]*)\|?([a-zA-Z0-9\.\-\/]*)/
export let trigger = function () { export let trigger = async () => {
let done = plugins.q.defer() plugins.beautylog.info('now running triggers')
plugins.beautylog.info('now running triggers') plugins.smartparam.forEachMinimatch(process.env, 'NPMCI_TRIGGER_*', evaluateTrigger)
plugins.smartparam.forEachMinimatch(process.env, 'NPMCI_TRIGGER_*', evaluateTrigger)
done.resolve()
return done.promise
} }
let evaluateTrigger = (triggerEnvVarArg) => { let evaluateTrigger = async (triggerEnvVarArg) => {
let triggerRegexResultArray = triggerValueRegex.exec(triggerEnvVarArg) let triggerRegexResultArray = triggerValueRegex.exec(triggerEnvVarArg)
let regexDomain = triggerRegexResultArray[1] let regexDomain = triggerRegexResultArray[1]
let regexProjectId = triggerRegexResultArray[2] let regexProjectId = triggerRegexResultArray[2]
let regexProjectTriggerToken = triggerRegexResultArray[3] let regexProjectTriggerToken = triggerRegexResultArray[3]
let regexRefName = triggerRegexResultArray[4] let regexRefName = triggerRegexResultArray[4]
let regexTriggerName let regexTriggerName
if (triggerRegexResultArray.length === 6) { if (triggerRegexResultArray.length === 6) {
regexTriggerName = triggerRegexResultArray[5] regexTriggerName = triggerRegexResultArray[5]
} else { } else {
regexTriggerName = 'Unnamed Trigger' regexTriggerName = 'Unnamed Trigger'
} }
plugins.beautylog.info('Found Trigger!') plugins.beautylog.info('Found Trigger!')
plugins.beautylog.log('triggering build for ref ' + regexRefName + ' of ' + regexTriggerName) plugins.beautylog.log('triggering build for ref ' + regexRefName + ' of ' + regexTriggerName)
plugins.request.post('https://gitlab.com/api/v3/projects/' + regexProjectId + '/trigger/builds', { form: { token: regexProjectTriggerToken, ref: regexRefName } }) plugins.request.post('https://gitlab.com/api/v3/projects/' + regexProjectId + '/trigger/builds', { form: { token: regexProjectTriggerToken, ref: regexRefName } })
} }

6
yarn.lock
View File

@ -1676,7 +1676,7 @@ oauth-sign@~0.8.1:
version "0.8.2" version "0.8.2"
resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43"
object-assign@4.1.0: object-assign@4.1.0, object-assign@^4.0.1:
version "4.1.0" version "4.1.0"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0"
@ -1684,10 +1684,6 @@ object-assign@^3.0.0:
version "3.0.0" version "3.0.0"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-3.0.0.tgz#9bedd5ca0897949bca47e7ff408062d549f587f2" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-3.0.0.tgz#9bedd5ca0897949bca47e7ff408062d549f587f2"
object-assign@^4.0.1:
version "4.1.1"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
object-component@0.0.3: object-component@0.0.3:
version "0.0.3" version "0.0.3"
resolved "https://registry.yarnpkg.com/object-component/-/object-component-0.0.3.tgz#f0c69aa50efc95b866c186f400a33769cb2f1291" resolved "https://registry.yarnpkg.com/object-component/-/object-component-0.0.3.tgz#f0c69aa50efc95b866c186f400a33769cb2f1291"