Compare commits

..

No commits in common. "master" and "v1.0.18" have entirely different histories.

16 changed files with 1716 additions and 4665 deletions

4
.gitignore vendored
View File

@ -15,6 +15,8 @@ node_modules/
# builds
dist/
dist_*/
dist_web/
dist_serve/
dist_ts_web/
# custom

View File

@ -4,7 +4,7 @@ image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
cache:
paths:
- .npmci_cache/
key: '$CI_BUILD_STAGE'
key: "$CI_BUILD_STAGE"
stages:
- security
@ -12,71 +12,64 @@ stages:
- release
- metadata
before_script:
- npm install -g @shipzone/npmci
# ====================
# security stage
# ====================
auditProductionDependencies:
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
mirror:
stage: security
script:
- npmci npm prepare
- npmci command npm install --production --ignore-scripts
- npmci command npm config set registry https://registry.npmjs.org
- npmci command npm audit --audit-level=high --only=prod --production
- npmci git mirror
tags:
- docker
allow_failure: true
- notpriv
auditDevDependencies:
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
snyk:
stage: security
script:
- npmci npm prepare
- npmci command npm install -g snyk
- npmci command npm install --ignore-scripts
- npmci command npm config set registry https://registry.npmjs.org
- npmci command npm audit --audit-level=high --only=dev
- npmci command snyk test
tags:
- docker
allow_failure: true
- notpriv
# ====================
# test stage
# ====================
testStable:
testLTS:
stage: test
script:
- npmci npm prepare
- npmci node install stable
- npmci node install lts
- npmci npm install
- npmci npm test
coverage: /\d+.?\d+?\%\s*coverage/
tags:
- docker
- notpriv
testBuild:
stage: test
script:
- npmci npm prepare
- npmci node install stable
- npmci node install lts
- npmci npm install
- npmci command npm run build
coverage: /\d+.?\d+?\%\s*coverage/
tags:
- docker
- notpriv
release:
stage: release
script:
- npmci node install stable
- npmci node install lts
- npmci npm publish
only:
- tags
tags:
- lossless
- docker
- notpriv
@ -86,14 +79,11 @@ release:
codequality:
stage: metadata
allow_failure: true
only:
- tags
script:
- npmci command npm install -g typescript
- npmci npm prepare
- npmci command npm install -g tslint typescript
- npmci npm install
- npmci command "tslint -c tslint.json ./ts/**/*.ts"
tags:
- lossless
- docker
- priv
@ -104,19 +94,18 @@ trigger:
only:
- tags
tags:
- lossless
- docker
- notpriv
pages:
image: hosttoday/ht-docker-node:npmci
stage: metadata
script:
- npmci node install stable
- npmci command npm install -g @gitzone/tsdoc
- npmci npm prepare
- npmci npm install
- npmci command npm run buildDocs
- npmci command tsdoc
tags:
- lossless
- docker
- notpriv
only:

4
.snyk Normal file
View File

@ -0,0 +1,4 @@
# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
version: v1.12.0
ignore: {}
patch: {}

11
.vscode/launch.json vendored
View File

@ -1,11 +0,0 @@
{
"version": "0.2.0",
"configurations": [
{
"command": "npm test",
"name": "Run npm test",
"request": "launch",
"type": "node-terminal"
}
]
}

26
.vscode/settings.json vendored
View File

@ -1,26 +0,0 @@
{
"json.schemas": [
{
"fileMatch": ["/npmextra.json"],
"schema": {
"type": "object",
"properties": {
"npmci": {
"type": "object",
"description": "settings for npmci"
},
"gitzone": {
"type": "object",
"description": "settings for gitzone",
"properties": {
"projectType": {
"type": "string",
"enum": ["website", "element", "service", "npm", "wcc"]
}
}
}
}
}
}
]
}

View File

@ -4,25 +4,13 @@
"npmAccessLevel": "public"
},
"gitzone": {
"projectType": "npm",
"module": {
"githost": "code.foss.global",
"gitscope": "push.rocks",
"githost": "gitlab.com",
"gitscope": "pushrocks",
"gitrepo": "smartcsv",
"description": "A module for handling CSV data compliant with Gitzone standard.",
"npmPackagename": "@push.rocks/smartcsv",
"license": "MIT",
"keywords": [
"CSV",
"data parsing",
"data conversion",
"TypeScript",
"CSV manipulation",
"CSV to JSON"
]
}
},
"tsdoc": {
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
"shortDescription": "handle csv data | gitzone standard compliant",
"npmPackagename": "@pushrocks/smartcsv",
"license": "MIT"
}
}
}

1588
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -1,56 +1,37 @@
{
"name": "@push.rocks/smartcsv",
"version": "2.0.2",
"name": "@pushrocks/smartcsv",
"version": "1.0.18",
"private": false,
"description": "A module for handling CSV data compliant with Gitzone standard.",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
"description": "handle csv data | gitzone standard compliant",
"main": "dist/index.js",
"typings": "dist/index.d.ts",
"author": "Lossless GmbH",
"license": "MIT",
"scripts": {
"test": "(tstest test/)",
"build": "(tsbuild --allowimplicitany)",
"buildDocs": "tsdoc"
"build": "(tsbuild)",
"format": "(gitzone format)"
},
"devDependencies": {
"@gitzone/tsbuild": "^2.1.65",
"@gitzone/tsrun": "^1.2.37",
"@gitzone/tstest": "^1.0.73",
"@pushrocks/smartfile": "^10.0.4",
"@pushrocks/tapbundle": "^5.0.4",
"@types/node": "^18.6.4"
"@gitzone/tsbuild": "^2.1.17",
"@gitzone/tsrun": "^1.2.8",
"@gitzone/tstest": "^1.0.28",
"@pushrocks/smartfile": "^7.0.6",
"@pushrocks/tapbundle": "^3.0.13",
"@types/node": "^12.7.12"
},
"dependencies": {
"@pushrocks/smartpromise": "^3.1.7",
"@pushrocks/smartstring": "^4.0.2"
"@pushrocks/smartpromise": "^3.0.6",
"@pushrocks/smartstring": "^3.0.14"
},
"files": [
"ts/**/*",
"ts_web/**/*",
"dist/**/*",
"dist_*/**/*",
"dist_ts/**/*",
"dist_ts_web/**/*",
"assets/**/*",
"ts/*",
"ts_web/*",
"dist/*",
"dist_web/*",
"assets/*",
"cli.js",
"npmextra.json",
"readme.md"
],
"browserslist": [
"last 1 chrome versions"
],
"keywords": [
"CSV",
"data parsing",
"data conversion",
"TypeScript",
"CSV manipulation",
"CSV to JSON"
],
"homepage": "https://code.foss.global/push.rocks/smartcsv",
"repository": {
"type": "git",
"url": "https://code.foss.global/push.rocks/smartcsv.git"
}
]
}

4341
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@ -1 +0,0 @@

128
readme.md
View File

@ -1,118 +1,26 @@
# @push.rocks/smartcsv
# @pushrocks/smartcsv
handle csv data | gitzone standard compliant
## Install
## Availabililty and Links
* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/smartcsv)
* [gitlab.com (source)](https://gitlab.com/pushrocks/smartcsv)
* [github.com (source mirror)](https://github.com/pushrocks/smartcsv)
* [docs (typedoc)](https://pushrocks.gitlab.io/smartcsv/)
To install `@push.rocks/smartcsv`, use the following command with npm:
```sh
npm install @push.rocks/smartcsv --save
```
Or, if you prefer using Yarn:
```sh
yarn add @push.rocks/smartcsv
```
## Status for master
[![build status](https://gitlab.com/pushrocks/smartcsv/badges/master/build.svg)](https://gitlab.com/pushrocks/smartcsv/commits/master)
[![coverage report](https://gitlab.com/pushrocks/smartcsv/badges/master/coverage.svg)](https://gitlab.com/pushrocks/smartcsv/commits/master)
[![npm downloads per month](https://img.shields.io/npm/dm/@pushrocks/smartcsv.svg)](https://www.npmjs.com/package/@pushrocks/smartcsv)
[![Known Vulnerabilities](https://snyk.io/test/npm/@pushrocks/smartcsv/badge.svg)](https://snyk.io/test/npm/@pushrocks/smartcsv)
[![TypeScript](https://img.shields.io/badge/TypeScript->=%203.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
[![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
[![JavaScript Style Guide](https://img.shields.io/badge/code%20style-prettier-ff69b4.svg)](https://prettier.io/)
## Usage
`@push.rocks/smartcsv` is a powerful library designed for handling CSV data efficiently and in a developer-friendly manner. This document will guide you through various use cases and demonstrate the flexibility and capabilities of `@push.rocks/smartcsv`.
For further information read the linked docs at the top of this readme.
### Importing the Module
> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy.html)
First, import `Csv` from the `@push.rocks/smartcsv` package into your TypeScript file:
```typescript
import { Csv } from '@push.rocks/smartcsv';
```
### Creating a CSV Instance from a String
You can create a `Csv` instance directly from a string. This is particularly useful when you have CSV data as a string from a file or an API response:
```typescript
const csvString = `Name, Age, Occupation\nJohn Doe, 28, Software Developer\nJane Smith, 32, Data Scientist`;
const csvOptions = {
headers: true, // indicates the first row contains headers
unquote: true, // unquote values if necessary
trimSpace: true, // trim spaces around values
removeBomMarkers: true // remove BOM markers if present
};
const csvInstance = await Csv.createCsvFromString(csvString, csvOptions);
```
### Exporting CSV Data as JSON
Once you have a `Csv` instance, you can convert the CSV data into JSON objects for easier manipulation and access:
```typescript
const jsonData = await csvInstance.exportAsObject();
console.log(jsonData);
```
This will output:
```json
[
{ "Name": "John Doe", "Age": "28", "Occupation": "Software Developer" },
{ "Name": "Jane Smith", "Age": "32", "Occupation": "Data Scientist" }
]
```
### Creating a CSV String from an Array of Objects
If you have an array of objects and wish to convert it into a CSV string, `@push.rocks/smartcsv` offers a straightforward approach:
```typescript
const arrayOfObjects = [
{ Name: "John Doe", Age: 28, Occupation: "Software Developer" },
{ Name: "Jane Smith", Age: 32, Occupation: "Data Scientist" }
];
const csvString = await Csv.createCsvStringFromArray(arrayOfObjects);
console.log(csvString);
```
This will generate a CSV string that represents the array of objects provided.
### Advanced Usage
#### Custom Separator Detection
By default, `@push.rocks/smartcsv` intelligently detects the separator used in the input CSV data (either a comma `,` or a semicolon `;`). You can rely on this automatic detection or specify preferred options for finer control over how your CSV data is handled.
#### Handling Quotes and BOM Markers
The library is capable of processing CSV files that include quoted fields, optionally removing quotes where necessary. Similarly, it can handle Byte Order Mark (BOM) markers at the start of UTF-8 encoded files, ensuring that the text is interpreted correctly.
#### Trimming Spaces
For data cleanliness, `@push.rocks/smartcsv` can trim leading and trailing spaces from values in the CSV, ensuring that the resulting data is neat and uniform.
### Conclusion
`@push.rocks/smartcsv` offers a comprehensive set of features for working with CSV data in TypeScript, providing flexibility, ease of use, and powerful data manipulation capabilities. Whether you're importing CSV data from a file, converting JSON to CSV, or manipulating CSV strings directly in your application, `@push.rocks/smartcsv` has you covered.
For detailed API documentation, please refer to the [official documentation](https://pushrocks.gitlab.io/smartcsv/).
## License and Legal Information
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
### Trademarks
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
### Company Information
Task Venture Capital GmbH
Registered at District court Bremen HRB 35230 HB, Germany
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
[![repo-footer](https://pushrocks.gitlab.io/assets/repo-footer.svg)](https://maintainedby.lossless.com)

View File

@ -1,12 +1,14 @@
// tslint:disable-next-line:no-implicit-dependencies
import { expect, tap } from '@pushrocks/tapbundle';
import * as smartcsv from '../ts/index.js';
import * as smartcsv from '../ts/index';
// tslint:disable-next-line:no-implicit-dependencies
import * as smartfile from '@pushrocks/smartfile';
let fileString: string;
let testCsv: smartcsv.Csv;
tap.test('should read a file', async (tools) => {
tap.test('should read a file', async tools => {
fileString = smartfile.fs.toStringSync('./test/sample.csv');
});
@ -19,7 +21,7 @@ tap.test('should create a valid csv', async () => {
tap.test('should create a valid csv string', async () => {
const createdCsvString = await smartcsv.Csv.createCsvStringFromArray([
{ wow: 'hi', wow2: 'there' },
{ wow: 'really', wow3: 'yes' },
{ wow: 'really', wow3: 'yes' }
]);
console.log(createdCsvString);
});

View File

@ -1,8 +0,0 @@
/**
* autocreated commitinfo by @pushrocks/commitinfo
*/
export const commitinfo = {
name: '@pushrocks/smartcsv',
version: '2.0.2',
description: 'handle csv data | gitzone standard compliant'
}

View File

@ -1,10 +1,8 @@
import * as plugins from './smartcsv.plugins.js';
import * as plugins from './smartcsv.plugins';
export interface ICsvConstructorOptions {
headers: boolean;
unquote?: boolean;
trimSpace?: boolean;
removeBomMarkers?: boolean;
unquote: boolean;
}
export class Csv {
@ -57,33 +55,23 @@ export class Csv {
public keyFrame: string = null;
public data: any[];
private base64RecognitionPrefix = '####12345####';
public options: ICsvConstructorOptions = {
headers: true,
unquote: true,
trimSpace: true,
removeBomMarkers: true,
unquote: true
};
constructor(csvStringArg: string, optionsArg: ICsvConstructorOptions) {
this.options = {
...this.options,
...optionsArg,
...optionsArg
};
let csvStringToParse = csvStringArg;
if (this.options.removeBomMarkers) {
csvStringToParse = csvStringToParse.replace(/^\uFEFF/, '');
}
// Quotes allow separators to be contained in quoted strings.
// To preserve them, when unquoting, we convert everything in between to base64 here
if (this.options.unquote) {
csvStringToParse = csvStringToParse.replace(
/"(.*?)"/gi,
(match, p1, offset, originalString) => {
return this.base64RecognitionPrefix + plugins.smartstring.base64.encode(match);
return plugins.smartstring.base64.encode(match);
}
);
}
@ -149,9 +137,8 @@ export class Csv {
this.headers = headerRow.split(this.keyFrame);
if (this.options.unquote) {
const unquotedHeaders: string[] = [];
for (let header of this.headers) {
if (header.startsWith(this.base64RecognitionPrefix)) {
header = header.replace(this.base64RecognitionPrefix, '');
for (const header of this.headers) {
if (plugins.smartstring.type.isBase64(header)) {
unquotedHeaders.push(plugins.smartstring.base64.decode(header).replace(/['"]+/g, ''));
} else {
unquotedHeaders.push(header);
@ -159,13 +146,6 @@ export class Csv {
}
this.headers = unquotedHeaders;
}
if (this.options.trimSpace) {
const trimmedHeaders = [];
for (const header of this.headers) {
trimmedHeaders.push(header.trim());
}
this.headers = trimmedHeaders;
}
}
return this.headers;
}
@ -175,15 +155,8 @@ export class Csv {
let resultJson: any = {};
for (let i = 0; i < neededIterations; i++) {
let value = dataArray[i];
if (this.options.unquote && value.startsWith(this.base64RecognitionPrefix)) {
value = value.replace(this.base64RecognitionPrefix, '');
value = plugins.smartstring.base64
.decode(value)
.replace(/['"]+/g, '')
.replace(/['"]+/g, '');
}
if (this.options.trimSpace) {
value = value.trim();
if (this.options.unquote && plugins.smartstring.type.isBase64(value)) {
value = plugins.smartstring.base64.decode(value).replace(/['"]+/g, '').replace(/['"]+/g, '');
}
resultJson[this.headers[i]] = value;
}

View File

@ -1,14 +0,0 @@
{
"compilerOptions": {
"experimentalDecorators": true,
"useDefineForClassFields": false,
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"esModuleInterop": true,
"verbatimModuleSyntax": true
},
"exclude": [
"dist_*/**/*.d.ts"
]
}

17
tslint.json Normal file
View File

@ -0,0 +1,17 @@
{
"extends": ["tslint:latest", "tslint-config-prettier"],
"rules": {
"semicolon": [true, "always"],
"no-console": false,
"ordered-imports": false,
"object-literal-sort-keys": false,
"member-ordering": {
"options":{
"order": [
"static-method"
]
}
}
},
"defaultSeverity": "warning"
}