fix(bom markers): now handles bom markers correctly

This commit is contained in:
Philipp Kunz 2022-01-03 17:37:44 +01:00
parent ab1c84f41f
commit fd0abaeb47
11 changed files with 26840 additions and 706 deletions

4
.gitignore vendored
View File

@ -15,8 +15,6 @@ node_modules/
# builds
dist/
dist_web/
dist_serve/
dist_ts_web/
dist_*/
# custom

View File

@ -12,6 +12,9 @@ stages:
- release
- metadata
before_script:
- npm install -g @shipzone/npmci
# ====================
# security stage
# ====================
@ -19,22 +22,36 @@ mirror:
stage: security
script:
- npmci git mirror
only:
- tags
tags:
- lossless
- docker
- notpriv
snyk:
image: registry.gitlab.com/hosttoday/ht-docker-node:snyk
auditProductionDependencies:
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
stage: security
script:
- npmci npm prepare
- npmci command npm install --production --ignore-scripts
- npmci command npm config set registry https://registry.npmjs.org
- npmci command npm audit --audit-level=high --only=prod --production
tags:
- docker
allow_failure: true
auditDevDependencies:
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
stage: security
script:
- npmci npm prepare
- npmci command npm install --ignore-scripts
- npmci command snyk test
- npmci command npm config set registry https://registry.npmjs.org
- npmci command npm audit --audit-level=high --only=dev
tags:
- lossless
- docker
- notpriv
allow_failure: true
# ====================
# test stage
@ -49,9 +66,7 @@ testStable:
- npmci npm test
coverage: /\d+.?\d+?\%\s*coverage/
tags:
- lossless
- docker
- priv
testBuild:
stage: test
@ -62,9 +77,7 @@ testBuild:
- npmci command npm run build
coverage: /\d+.?\d+?\%\s*coverage/
tags:
- lossless
- docker
- notpriv
release:
stage: release
@ -84,6 +97,8 @@ release:
codequality:
stage: metadata
allow_failure: true
only:
- tags
script:
- npmci command npm install -g tslint typescript
- npmci npm prepare

4
.snyk
View File

@ -1,4 +0,0 @@
# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
version: v1.12.0
ignore: {}
patch: {}

11
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,11 @@
{
"version": "0.2.0",
"configurations": [
{
"command": "npm test",
"name": "Run npm test",
"request": "launch",
"type": "node-terminal"
}
]
}

26
.vscode/settings.json vendored Normal file
View File

@ -0,0 +1,26 @@
{
"json.schemas": [
{
"fileMatch": ["/npmextra.json"],
"schema": {
"type": "object",
"properties": {
"npmci": {
"type": "object",
"description": "settings for npmci"
},
"gitzone": {
"type": "object",
"description": "settings for gitzone",
"properties": {
"projectType": {
"type": "string",
"enum": ["website", "element", "service", "npm", "wcc"]
}
}
}
}
}
}
]
}

View File

@ -4,6 +4,7 @@
"npmAccessLevel": "public"
},
"gitzone": {
"projectType": "npm",
"module": {
"githost": "gitlab.com",
"gitscope": "pushrocks",

27348
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -3,38 +3,41 @@
"version": "1.0.22",
"private": false,
"description": "handle csv data | gitzone standard compliant",
"main": "dist/index.js",
"typings": "dist/index.d.ts",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"author": "Lossless GmbH",
"license": "MIT",
"scripts": {
"test": "(tstest test/)",
"build": "(tsbuild)",
"format": "(gitzone format)"
"build": "(tsbuild)"
},
"devDependencies": {
"@gitzone/tsbuild": "^2.1.17",
"@gitzone/tsrun": "^1.2.8",
"@gitzone/tstest": "^1.0.28",
"@pushrocks/smartfile": "^7.0.6",
"@pushrocks/tapbundle": "^3.2.0",
"@types/node": "^12.12.17",
"tslint": "^6.0.0-beta0",
"@gitzone/tsbuild": "^2.1.28",
"@gitzone/tsrun": "^1.2.18",
"@gitzone/tstest": "^1.0.60",
"@pushrocks/smartfile": "^9.0.5",
"@pushrocks/tapbundle": "^3.2.15",
"@types/node": "^17.0.7",
"tslint": "^6.1.3",
"tslint-config-prettier": "^1.18.0"
},
"dependencies": {
"@pushrocks/smartpromise": "^3.0.6",
"@pushrocks/smartstring": "^3.0.17"
"@pushrocks/smartpromise": "^3.1.6",
"@pushrocks/smartstring": "^3.0.24"
},
"files": [
"ts/**/*",
"ts_web/**/*",
"dist/**/*",
"dist_web/**/*",
"dist_*/**/*",
"dist_ts/**/*",
"dist_ts_web/**/*",
"assets/**/*",
"cli.js",
"npmextra.json",
"readme.md"
],
"browserslist": [
"last 1 chrome versions"
]
}

View File

@ -8,16 +8,27 @@ handle csv data | gitzone standard compliant
* [docs (typedoc)](https://pushrocks.gitlab.io/smartcsv/)
## Status for master
[![build status](https://gitlab.com/pushrocks/smartcsv/badges/master/build.svg)](https://gitlab.com/pushrocks/smartcsv/commits/master)
[![coverage report](https://gitlab.com/pushrocks/smartcsv/badges/master/coverage.svg)](https://gitlab.com/pushrocks/smartcsv/commits/master)
[![npm downloads per month](https://img.shields.io/npm/dm/@pushrocks/smartcsv.svg)](https://www.npmjs.com/package/@pushrocks/smartcsv)
[![Known Vulnerabilities](https://snyk.io/test/npm/@pushrocks/smartcsv/badge.svg)](https://snyk.io/test/npm/@pushrocks/smartcsv)
[![TypeScript](https://img.shields.io/badge/TypeScript->=%203.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
[![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
[![JavaScript Style Guide](https://img.shields.io/badge/code%20style-prettier-ff69b4.svg)](https://prettier.io/)
Status Category | Status Badge
-- | --
GitLab Pipelines | [![pipeline status](https://gitlab.com/pushrocks/smartcsv/badges/master/pipeline.svg)](https://lossless.cloud)
GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/pushrocks/smartcsv/badges/master/coverage.svg)](https://lossless.cloud)
npm | [![npm downloads per month](https://badgen.net/npm/dy/@pushrocks/smartcsv)](https://lossless.cloud)
Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/pushrocks/smartcsv)](https://lossless.cloud)
TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud)
node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud)
PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@pushrocks/smartcsv)](https://lossless.cloud)
PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@pushrocks/smartcsv)](https://lossless.cloud)
BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@pushrocks/smartcsv)](https://lossless.cloud)
Platform support | [![Supports Windows 10](https://badgen.net/badge/supports%20Windows%2010/yes/green?icon=windows)](https://lossless.cloud) [![Supports Mac OS X](https://badgen.net/badge/supports%20Mac%20OS%20X/yes/green?icon=apple)](https://lossless.cloud)
## Usage
## Contribution
We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
## Contribution

View File

@ -1,14 +1,12 @@
// tslint:disable-next-line:no-implicit-dependencies
import { expect, tap } from '@pushrocks/tapbundle';
import * as smartcsv from '../ts/index';
// tslint:disable-next-line:no-implicit-dependencies
import * as smartfile from '@pushrocks/smartfile';
let fileString: string;
let testCsv: smartcsv.Csv;
tap.test('should read a file', async tools => {
tap.test('should read a file', async (tools) => {
fileString = smartfile.fs.toStringSync('./test/sample.csv');
});
@ -21,7 +19,7 @@ tap.test('should create a valid csv', async () => {
tap.test('should create a valid csv string', async () => {
const createdCsvString = await smartcsv.Csv.createCsvStringFromArray([
{ wow: 'hi', wow2: 'there' },
{ wow: 'really', wow3: 'yes' }
{ wow: 'really', wow3: 'yes' },
]);
console.log(createdCsvString);
});

View File

@ -3,6 +3,7 @@ import * as plugins from './smartcsv.plugins';
export interface ICsvConstructorOptions {
headers: boolean;
unquote?: boolean;
removeBomMarkers?: boolean;
}
export class Csv {
@ -59,16 +60,20 @@ export class Csv {
public options: ICsvConstructorOptions = {
headers: true,
unquote: true
unquote: true,
removeBomMarkers: true,
};
constructor(csvStringArg: string, optionsArg: ICsvConstructorOptions) {
this.options = {
...this.options,
...optionsArg
...optionsArg,
};
let csvStringToParse = csvStringArg;
if (this.options.removeBomMarkers) {
csvStringToParse = csvStringToParse.replace(/^\uFEFF/, '');
}
if (this.options.unquote) {
csvStringToParse = csvStringToParse.replace(
/"(.*?)"/gi,