This repository has been archived by the owner on Jun 3, 2020. It is now read-only.
-
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #57 from Borewit/tokenizer-s3
Utilize @tokenizer/s3
- Loading branch information
Showing
5 changed files
with
47 additions
and
119 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,78 +1,20 @@ | ||
| import * as S3 from 'aws-sdk/clients/s3'; | ||
| import { parseFromTokenizer, parseStream } from 'music-metadata/lib/core'; | ||
| import { RangeRequestTokenizer, IRangeRequestClient, IRangeRequestResponse, parseContentRange } from '@tokenizer/range'; | ||
| import { AWSError, Request } from 'aws-sdk'; | ||
| import { IOptions, IAudioMetadata } from 'music-metadata/lib/type'; | ||
| import { makeTokenizer, IS3Options} from '@tokenizer/s3'; | ||
| import * as mm from 'music-metadata/lib/core'; | ||
|
|
||
| export { IPicture, IAudioMetadata, IOptions, ITag, INativeTagDict } from 'music-metadata/lib/type'; | ||
|
|
||
| interface IS3Options extends IOptions { | ||
| /** | ||
| * Flag to disable chunked transfer, use conventional HTTPS stream instead | ||
| */ | ||
| disableChunked?: boolean; | ||
| interface IMMS3Options extends IOptions, IS3Options { | ||
| } | ||
|
|
||
| /** | ||
| * Use S3-client to execute actual HTTP-requests. | ||
| * Retrieve metadata from Amazon S3 object | ||
| * @param objRequest S3 object request | ||
| * @param options music-metadata options | ||
| * @return Metadata | ||
| */ | ||
| class S3Request implements IRangeRequestClient { | ||
|
|
||
| constructor(private s3client: MMS3Client, private objRequest: S3.Types.GetObjectRequest) { | ||
| } | ||
|
|
||
| public async getResponse(method, range: number[]): Promise<IRangeRequestResponse> { | ||
|
|
||
| return this.s3client.getRangedRequest(this.objRequest, range).promise().then(data => { | ||
| return { | ||
| contentLength: data.ContentLength, | ||
| contentType: data.ContentType, | ||
| contentRange: parseContentRange(data.ContentRange), | ||
| arrayBuffer: async () => { | ||
| return data.Body as Buffer; | ||
| } | ||
| }; | ||
| }); | ||
| } | ||
| } | ||
|
|
||
| export class MMS3Client { | ||
|
|
||
| constructor(private s3: S3) { | ||
| } | ||
|
|
||
| /** | ||
| * Do a ranged request, this method will be called by streaming-http-token-reader | ||
| * @param objRequest | ||
| * @param range | ||
| */ | ||
| public getRangedRequest(objRequest: S3.Types.GetObjectRequest, range: number[]): Request<S3.Types.GetObjectOutput, AWSError> { | ||
| const rangedRequest = {...objRequest}; // Copy request | ||
| rangedRequest.Range = `bytes=${range[0]}-${range[1]}`; | ||
| return this.s3.getObject(rangedRequest); | ||
| } | ||
|
|
||
| /** | ||
| * Retrieve metadata from Amazon S3 object | ||
| * @param objRequest S3 object request | ||
| * @param options music-metadata options | ||
| */ | ||
| public async parseS3Object(objRequest: S3.Types.GetObjectRequest, options?: IS3Options): Promise<IAudioMetadata> { | ||
| if (options && options.disableChunked) { | ||
|
|
||
| const info = await this.getRangedRequest(objRequest, [0, 0]).promise(); | ||
|
|
||
| const stream = this.s3 | ||
| .getObject(objRequest) | ||
| .createReadStream(); | ||
| return parseStream(stream, info.ContentType, options); | ||
| } else { | ||
| const s3Request = new S3Request(this, objRequest); | ||
| const rangeRequestTokenizer = new RangeRequestTokenizer(s3Request, { | ||
| avoidHeadRequests: true | ||
| }); | ||
| await rangeRequestTokenizer.init(); | ||
| return parseFromTokenizer(rangeRequestTokenizer, rangeRequestTokenizer.contentType, options); | ||
| } | ||
| } | ||
| export async function parseS3Object(s3: S3, objRequest: S3.Types.GetObjectRequest, options?: IMMS3Options): Promise<IAudioMetadata> { | ||
| const s3Tokenizer = await makeTokenizer(s3, objRequest, options); | ||
| return mm.parseFromTokenizer(s3Tokenizer, options); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -39,13 +39,21 @@ | |
| "@nodelib/fs.scandir" "2.1.3" | ||
| fastq "^1.6.0" | ||
|
|
||
| "@tokenizer/range@^0.1.0": | ||
| version "0.1.1" | ||
| resolved "https://registry.yarnpkg.com/@tokenizer/range/-/range-0.1.1.tgz#a174031cbdb8eebf3dc0e0a8eee41217e42d8609" | ||
| integrity sha512-GpRSPWZt/AtCjwE1N4m+wFI0r1L7VhLpXiRXaohL3D5QSkUXbl0Im0BzEaxhz7UPg5eT/BxlUwUHMGacwQrTeA== | ||
| "@tokenizer/range@^0.2.1": | ||
| version "0.2.1" | ||
| resolved "https://registry.yarnpkg.com/@tokenizer/range/-/range-0.2.1.tgz#f50605c2b42936f296d8fd32569fee5c79ee6bbf" | ||
| integrity sha512-guIiNyR/VXFtYbpMbfQj8zVdbwu5MoRjh2YE/NwFeJJwkBOPBw5FhoHmJ7IAv4btHlFeBGtRDiLuuZtYkr12AA== | ||
| dependencies: | ||
| debug "^4.1.1" | ||
| strtok3 "^3.1.4" | ||
| strtok3 "^5.0.0" | ||
|
|
||
| "@tokenizer/s3@^0.1.1": | ||
| version "0.1.1" | ||
| resolved "https://registry.yarnpkg.com/@tokenizer/s3/-/s3-0.1.1.tgz#4b55e9f0cf64ae1badfe2d5b6dd7169a60cba4e4" | ||
| integrity sha512-4mhhe8/aQBfjeNC3xH3SAx/IY4zrO0S2/HFbsHJWoWQtPZIU8UVgdfl/nm4197EZLMtPwO+i0k1c8cc//V6Fng== | ||
| dependencies: | ||
| "@tokenizer/range" "^0.2.1" | ||
| strtok3 "^5.0.0" | ||
|
|
||
| "@tokenizer/token@^0.1.0": | ||
| version "0.1.0" | ||
|
|
@@ -867,16 +875,16 @@ ms@^2.1.1: | |
| resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" | ||
| integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== | ||
|
|
||
| music-metadata@^5.4.3: | ||
| version "5.4.3" | ||
| resolved "https://registry.yarnpkg.com/music-metadata/-/music-metadata-5.4.3.tgz#b13c22e59312788922b0656e45a181075148cf6b" | ||
| integrity sha512-jGEfhqgfs3omD9sqaQg9ZzYXxsYYqv9wxMWz40WTNtm9cY7NE9TX+y+cL/JCs2Bs3+rj5xokOn6ZddpVvGsVFg== | ||
| music-metadata@^6.0.1: | ||
| version "6.0.1" | ||
| resolved "https://registry.yarnpkg.com/music-metadata/-/music-metadata-6.0.1.tgz#968c25f3a1cb682d80fe5406a1bb180349f9b1b0" | ||
| integrity sha512-UrscSZs7MBhiGZZQgZHJj9tHdhl2+DLLC0jHhFyoHKUrrTg5SKFlVYmW76NgOk70hSpGiNN4x8s/izVMOs8hyg== | ||
| dependencies: | ||
| content-type "^1.0.4" | ||
| debug "^4.1.0" | ||
| file-type "^12.4.2" | ||
| media-typer "^1.1.0" | ||
| strtok3 "^4.1.1" | ||
| strtok3 "^5.0.0" | ||
| token-types "^2.0.0" | ||
|
|
||
| [email protected]: | ||
|
|
@@ -1224,18 +1232,10 @@ [email protected]: | |
| resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" | ||
| integrity sha1-PFMZQukIwml8DsNEhYwobHygpgo= | ||
|
|
||
| strtok3@^3.1.4: | ||
| version "3.1.7" | ||
| resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-3.1.7.tgz#e236e955e7c93c156442a06f7b385a90a7f528d4" | ||
| integrity sha512-N0qFNf5vkyXgaPr4AHRmCTqdFwTnB1LxX3xo1N5KHMLhqircDdL9m8qem3Lpb8IIuJaNL1Jddazvb9WRwUdSlw== | ||
| dependencies: | ||
| debug "^4.1.1" | ||
| then-read-stream "^2.0.8" | ||
|
|
||
| strtok3@^4.1.1: | ||
| version "4.1.1" | ||
| resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-4.1.1.tgz#75043bb6175ebb22f10d48dfe9b06560345dc647" | ||
| integrity sha512-7nfDPVwCrx35LVYqEZPfrNJuoqlgOcsW2PIcru4/IbYXjtI17WtdZLtRJtpwR1Mj/alJ01FY57NsX4Gwl/ntTg== | ||
| strtok3@^5.0.0: | ||
| version "5.0.0" | ||
| resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-5.0.0.tgz#cf48cdede0b7641eb97e62918abce18441cff688" | ||
| integrity sha512-HpdgEUSkMqlTjO7uWEBvWHEKBYqXCbVeihlE+sa0keGsfVXspVxye1dPa4OYvnzOJsErzn6ohQU1U/ozcVAPKQ== | ||
| dependencies: | ||
| "@tokenizer/token" "^0.1.0" | ||
| debug "^4.1.1" | ||
|
|
@@ -1255,11 +1255,6 @@ supports-color@^5.3.0: | |
| dependencies: | ||
| has-flag "^3.0.0" | ||
|
|
||
| then-read-stream@^2.0.8: | ||
| version "2.0.8" | ||
| resolved "https://registry.yarnpkg.com/then-read-stream/-/then-read-stream-2.0.8.tgz#1d2c29e46af1327101875abef183e43313876c1b" | ||
| integrity sha512-OIQn3/zF2J/gp6mAQTbBb1AR+3yoSRqjaij0gGnEUcTl93T840mWIZ9sJWwubjwP7VUDwJpT+Tdl7T9RrQmMlw== | ||
|
|
||
| then-read-stream@^3.0.0: | ||
| version "3.0.0" | ||
| resolved "https://registry.yarnpkg.com/then-read-stream/-/then-read-stream-3.0.0.tgz#4a4ec37e23f18135b56fbc61670b6e8195e545b2" | ||
|
|
||