Google Picker #5443
Open
Google Picker #5443
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Diff output filesdiff --git a/packages/@uppy/companion-client/lib/tokenStorage.js b/packages/@uppy/companion-client/lib/tokenStorage.js
index df49432..26bb521 100644
--- a/packages/@uppy/companion-client/lib/tokenStorage.js
+++ b/packages/@uppy/companion-client/lib/tokenStorage.js
@@ -1,15 +1,9 @@
-export function setItem(key, value) {
- return new Promise(resolve => {
- localStorage.setItem(key, value);
- resolve();
- });
+export async function setItem(key, value) {
+ localStorage.setItem(key, value);
}
-export function getItem(key) {
- return Promise.resolve(localStorage.getItem(key));
+export async function getItem(key) {
+ return localStorage.getItem(key);
}
-export function removeItem(key) {
- return new Promise(resolve => {
- localStorage.removeItem(key);
- resolve();
- });
+export async function removeItem(key) {
+ localStorage.removeItem(key);
}
diff --git a/packages/@uppy/companion/lib/companion.js b/packages/@uppy/companion/lib/companion.js
index 9416e5e..333b8b9 100644
--- a/packages/@uppy/companion/lib/companion.js
+++ b/packages/@uppy/companion/lib/companion.js
@@ -11,6 +11,7 @@ const providerManager = require("./server/provider");
const controllers = require("./server/controllers");
const s3 = require("./server/controllers/s3");
const url = require("./server/controllers/url");
+const googlePicker = require("./server/controllers/googlePicker");
const createEmitter = require("./server/emitter");
const redis = require("./server/redis");
const jobs = require("./server/jobs");
@@ -107,6 +108,9 @@ module.exports.app = (optionsArg = {}) => {
if (options.enableUrlEndpoint) {
app.use("/url", url());
}
+ if (options.enableGooglePickerEndpoint) {
+ app.use("/google-picker", googlePicker());
+ }
app.post(
"/:providerName/preauth",
express.json(),
diff --git a/packages/@uppy/companion/lib/config/companion.d.ts b/packages/@uppy/companion/lib/config/companion.d.ts
index 3924a64..9aff10f 100644
--- a/packages/@uppy/companion/lib/config/companion.d.ts
+++ b/packages/@uppy/companion/lib/config/companion.d.ts
@@ -12,6 +12,7 @@ export namespace defaultOptions {
export let expires: number;
}
let enableUrlEndpoint: boolean;
+ let enableGooglePickerEndpoint: boolean;
let allowLocalUrls: boolean;
let periodicPingUrls: any[];
let streamingUpload: boolean;
diff --git a/packages/@uppy/companion/lib/config/companion.js b/packages/@uppy/companion/lib/config/companion.js
index 542f378..5892411 100644
--- a/packages/@uppy/companion/lib/config/companion.js
+++ b/packages/@uppy/companion/lib/config/companion.js
@@ -18,6 +18,7 @@ const defaultOptions = {
expires: 800, // seconds
},
enableUrlEndpoint: false,
+ enableGooglePickerEndpoint: false,
allowLocalUrls: false,
periodicPingUrls: [],
streamingUpload: true,
diff --git a/packages/@uppy/companion/lib/server/controllers/url.js b/packages/@uppy/companion/lib/server/controllers/url.js
index 3e0eff3..6a0b3be 100644
--- a/packages/@uppy/companion/lib/server/controllers/url.js
+++ b/packages/@uppy/companion/lib/server/controllers/url.js
@@ -2,35 +2,15 @@
Object.defineProperty(exports, "__esModule", { value: true });
const express = require("express");
const { startDownUpload } = require("../helpers/upload");
-const { prepareStream } = require("../helpers/utils");
+const { downloadURL } = require("../download");
const { validateURL } = require("../helpers/request");
-const { getURLMeta, getProtectedGot } = require("../helpers/request");
+const { getURLMeta } = require("../helpers/request");
const logger = require("../logger");
/**
* @callback downloadCallback
* @param {Error} err
* @param {string | Buffer | Buffer[]} chunk
*/
-/**
- * Downloads the content in the specified url, and passes the data
- * to the callback chunk by chunk.
- *
- * @param {string} url
- * @param {boolean} allowLocalIPs
- * @param {string} traceId
- * @returns {Promise}
- */
-const downloadURL = async (url, allowLocalIPs, traceId) => {
- try {
- const protectedGot = await getProtectedGot({ allowLocalIPs });
- const stream = protectedGot.stream.get(url, { responseType: "json" });
- const { size } = await prepareStream(stream);
- return { stream, size };
- } catch (err) {
- logger.error(err, "controller.url.download.error", traceId);
- throw err;
- }
-};
/**
* Fetches the size and content type of a URL
*
diff --git a/packages/@uppy/companion/lib/server/helpers/request.d.ts b/packages/@uppy/companion/lib/server/helpers/request.d.ts
index eff658a..69a7536 100644
--- a/packages/@uppy/companion/lib/server/helpers/request.d.ts
+++ b/packages/@uppy/companion/lib/server/helpers/request.d.ts
@@ -1,5 +1,5 @@
/// <reference types="node" />
-export function getURLMeta(url: string, allowLocalIPs?: boolean): Promise<{
+export function getURLMeta(url: string, allowLocalIPs?: boolean, options?: any): Promise<{
name: string;
type: string;
size: number;
diff --git a/packages/@uppy/companion/lib/server/helpers/request.js b/packages/@uppy/companion/lib/server/helpers/request.js
index 9c78f29..fc2bd1b 100644
--- a/packages/@uppy/companion/lib/server/helpers/request.js
+++ b/packages/@uppy/companion/lib/server/helpers/request.js
@@ -94,10 +94,10 @@ module.exports.getProtectedGot = getProtectedGot;
* @param {boolean} allowLocalIPs
* @returns {Promise<{name: string, type: string, size: number}>}
*/
-exports.getURLMeta = async (url, allowLocalIPs = false) => {
+exports.getURLMeta = async (url, allowLocalIPs = false, options = undefined) => {
async function requestWithMethod(method) {
const protectedGot = await getProtectedGot({ allowLocalIPs });
- const stream = protectedGot.stream(url, { method, throwHttpErrors: false });
+ const stream = protectedGot.stream(url, { method, throwHttpErrors: false, ...options });
return new Promise((resolve, reject) => (stream
.on("response", (response) => {
// Can be undefined for unknown length URLs, e.g. transfer-encoding: chunked
diff --git a/packages/@uppy/companion/lib/server/provider/google/drive/index.d.ts b/packages/@uppy/companion/lib/server/provider/google/drive/index.d.ts
index a13c57b..8ac298f 100644
--- a/packages/@uppy/companion/lib/server/provider/google/drive/index.d.ts
+++ b/packages/@uppy/companion/lib/server/provider/google/drive/index.d.ts
@@ -1,10 +1,9 @@
-export = Drive;
/**
* Adapter for API https://developers.google.com/drive/api/v3/
*/
-declare class Drive extends Provider {
+export class Drive extends Provider {
list(options: any): Promise<any>;
- download({ id: idIn, token }: {
+ download({ id, token }: {
id: any;
token: any;
}): Promise<any>;
@@ -15,6 +14,16 @@ declare class Drive extends Provider {
logout: typeof logout;
refreshToken: typeof refreshToken;
}
+export function streamGoogleFile({ token, id: idIn }: {
+ token: any;
+ id: any;
+}): Promise<{
+ stream: import("got", { with: { "resolution-mode": "import" } }).Request;
+}>;
+export function getGoogleFileSize({ id, token }: {
+ id: any;
+ token: any;
+}): Promise<number>;
import Provider = require("../../Provider");
import { logout } from "../index";
import { refreshToken } from "../index";
diff --git a/packages/@uppy/companion/lib/server/provider/google/drive/index.js b/packages/@uppy/companion/lib/server/provider/google/drive/index.js
index e5bcbf2..d48a5cd 100644
--- a/packages/@uppy/companion/lib/server/provider/google/drive/index.js
+++ b/packages/@uppy/companion/lib/server/provider/google/drive/index.js
@@ -43,6 +43,49 @@ async function getStats({ id, token }) {
}
return stats;
}
+async function streamGoogleFile({ token, id: idIn }) {
+ const client = await getClient({ token });
+ const { mimeType, id, exportLinks } = await getStats({ id: idIn, token });
+ let stream;
+ if (isGsuiteFile(mimeType)) {
+ const mimeType2 = getGsuiteExportType(mimeType);
+ logger.info(`calling google file export for ${id} to ${mimeType2}`, "provider.drive.export");
+ // GSuite files exported with large converted size results in error using standard export method.
+ // Error message: "This file is too large to be exported.".
+ // Issue logged in Google APIs: https://github.com/googleapis/google-api-nodejs-client/issues/3446
+ // Implemented based on the answer from StackOverflow: https://stackoverflow.com/a/59168288
+ const mimeTypeExportLink = exportLinks?.[mimeType2];
+ if (mimeTypeExportLink) {
+ const gSuiteFilesClient = (await got).extend({
+ headers: {
+ authorization: `Bearer ${token}`,
+ },
+ });
+ stream = gSuiteFilesClient.stream.get(mimeTypeExportLink, { responseType: "json" });
+ } else {
+ stream = client.stream.get(`files/${encodeURIComponent(id)}/export`, {
+ searchParams: { supportsAllDrives: true, mimeType: mimeType2 },
+ responseType: "json",
+ });
+ }
+ } else {
+ stream = client.stream.get(`files/${encodeURIComponent(id)}`, {
+ searchParams: { alt: "media", supportsAllDrives: true },
+ responseType: "json",
+ });
+ }
+ await prepareStream(stream);
+ return { stream };
+}
+async function getGoogleFileSize({ id, token }) {
+ const { mimeType, size } = await getStats({ id, token });
+ if (isGsuiteFile(mimeType)) {
+ // GSuite file sizes cannot be predetermined (but are max 10MB)
+ // e.g. Transfer-Encoding: chunked
+ return undefined;
+ }
+ return parseInt(size, 10);
+}
/**
* Adapter for API https://developers.google.com/drive/api/v3/
*/
@@ -115,7 +158,7 @@ class Drive extends Provider {
});
}
// eslint-disable-next-line class-methods-use-this
- async download({ id: idIn, token }) {
+ async download({ id, token }) {
if (mockAccessTokenExpiredError != null) {
logger.warn(`Access token: ${token}`);
if (mockAccessTokenExpiredError === token) {
@@ -124,53 +167,22 @@ class Drive extends Provider {
}
}
return withGoogleErrorHandling(Drive.oauthProvider, "provider.drive.download.error", async () => {
- const client = await getClient({ token });
- const { mimeType, id, exportLinks } = await getStats({ id: idIn, token });
- let stream;
- if (isGsuiteFile(mimeType)) {
- const mimeType2 = getGsuiteExportType(mimeType);
- logger.info(`calling google file export for ${id} to ${mimeType2}`, "provider.drive.export");
- // GSuite files exported with large converted size results in error using standard export method.
- // Error message: "This file is too large to be exported.".
- // Issue logged in Google APIs: https://github.com/googleapis/google-api-nodejs-client/issues/3446
- // Implemented based on the answer from StackOverflow: https://stackoverflow.com/a/59168288
- const mimeTypeExportLink = exportLinks?.[mimeType2];
- if (mimeTypeExportLink) {
- const gSuiteFilesClient = (await got).extend({
- headers: {
- authorization: `Bearer ${token}`,
- },
- });
- stream = gSuiteFilesClient.stream.get(mimeTypeExportLink, { responseType: "json" });
- } else {
- stream = client.stream.get(`files/${encodeURIComponent(id)}/export`, {
- searchParams: { supportsAllDrives: true, mimeType: mimeType2 },
- responseType: "json",
- });
- }
- } else {
- stream = client.stream.get(`files/${encodeURIComponent(id)}`, {
- searchParams: { alt: "media", supportsAllDrives: true },
- responseType: "json",
- });
- }
- await prepareStream(stream);
- return { stream };
+ return streamGoogleFile({ token, id });
});
}
// eslint-disable-next-line class-methods-use-this
async size({ id, token }) {
- return withGoogleErrorHandling(Drive.oauthProvider, "provider.drive.size.error", async () => {
- const { mimeType, size } = await getStats({ id, token });
- if (isGsuiteFile(mimeType)) {
- // GSuite file sizes cannot be predetermined (but are max 10MB)
- // e.g. Transfer-Encoding: chunked
- return undefined;
- }
- return parseInt(size, 10);
- });
+ return withGoogleErrorHandling(
+ Drive.oauthProvider,
+ "provider.drive.size.error",
+ async () => (getGoogleFileSize({ id, token })),
+ );
}
}
Drive.prototype.logout = logout;
Drive.prototype.refreshToken = refreshToken;
-module.exports = Drive;
+module.exports = {
+ Drive,
+ streamGoogleFile,
+ getGoogleFileSize,
+};
diff --git a/packages/@uppy/companion/lib/server/provider/index.js b/packages/@uppy/companion/lib/server/provider/index.js
index 7691d83..12e8acb 100644
--- a/packages/@uppy/companion/lib/server/provider/index.js
+++ b/packages/@uppy/companion/lib/server/provider/index.js
@@ -5,7 +5,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
*/
const dropbox = require("./dropbox");
const box = require("./box");
-const drive = require("./google/drive");
+const { Drive } = require("./google/drive");
const googlephotos = require("./google/googlephotos");
const instagram = require("./instagram/graph");
const facebook = require("./facebook");
@@ -65,7 +65,7 @@ module.exports.getProviderMiddleware = (providers, grantConfig) => {
* @returns {Record<string, typeof Provider>}
*/
module.exports.getDefaultProviders = () => {
- const providers = { dropbox, box, drive, googlephotos, facebook, onedrive, zoom, instagram, unsplash };
+ const providers = { dropbox, box, drive: Drive, googlephotos, facebook, onedrive, zoom, instagram, unsplash };
return providers;
};
/**
diff --git a/packages/@uppy/companion/lib/standalone/helper.js b/packages/@uppy/companion/lib/standalone/helper.js
index edaa622..ccc5e1a 100644
--- a/packages/@uppy/companion/lib/standalone/helper.js
+++ b/packages/@uppy/companion/lib/standalone/helper.js
@@ -143,6 +143,7 @@ const getConfigFromEnv = () => {
validHosts,
},
enableUrlEndpoint: process.env.COMPANION_ENABLE_URL_ENDPOINT === "true",
+ enableGooglePickerEndpoint: process.env.COMPANION_ENABLE_GOOGLE_PICKER_ENDPOINT === "true",
periodicPingUrls: process.env.COMPANION_PERIODIC_PING_URLS
? process.env.COMPANION_PERIODIC_PING_URLS.split(",")
: [],
diff --git a/packages/@uppy/core/lib/locale.js b/packages/@uppy/core/lib/locale.js
index 7311693..6ac4428 100644
--- a/packages/@uppy/core/lib/locale.js
+++ b/packages/@uppy/core/lib/locale.js
@@ -36,6 +36,9 @@ export default {
openFolderNamed: "Open folder %{name}",
cancel: "Cancel",
logOut: "Log out",
+ logIn: "Log in",
+ pickFiles: "Pick files",
+ pickPhotos: "Pick photos",
filter: "Filter",
resetFilter: "Reset filter",
loading: "Loading...",
@@ -56,5 +59,6 @@ export default {
},
additionalRestrictionsFailed: "%{count} additional restrictions were not fulfilled",
unnamed: "Unnamed",
+ pleaseWait: "Please wait",
},
};
diff --git a/packages/@uppy/provider-views/lib/index.js b/packages/@uppy/provider-views/lib/index.js
index 3e5b730..9d7cc3d 100644
--- a/packages/@uppy/provider-views/lib/index.js
+++ b/packages/@uppy/provider-views/lib/index.js
@@ -1,2 +1,3 @@
+export { default as GooglePickerView } from "./GooglePicker/GooglePickerView.js";
export { default as ProviderViews, defaultPickerIcon } from "./ProviderView/index.js";
export { default as SearchProviderViews } from "./SearchProviderView/index.js";
diff --git a/packages/@uppy/transloadit/lib/index.js b/packages/@uppy/transloadit/lib/index.js
index 2e8d13d..3a923d3 100644
--- a/packages/@uppy/transloadit/lib/index.js
+++ b/packages/@uppy/transloadit/lib/index.js
@@ -459,6 +459,8 @@ function _getClientVersion2() {
addPluginVersion("Facebook", "uppy-facebook");
addPluginVersion("GoogleDrive", "uppy-google-drive");
addPluginVersion("GooglePhotos", "uppy-google-photos");
+ addPluginVersion("GoogleDrivePicker", "uppy-google-drive-picker");
+ addPluginVersion("GooglePhotosPicker", "uppy-google-photos-picker");
addPluginVersion("Instagram", "uppy-instagram");
addPluginVersion("OneDrive", "uppy-onedrive");
addPluginVersion("Zoom", "uppy-zoom"); |
2 tasks
StrixOSG
reviewed
Nov 13, 2024
Comment on lines
97
to
98
| // NOTE: photos is broken and results in an error being returned from Google | ||
| // .addView(google.picker.ViewId.PHOTOS) |
There was a problem hiding this comment.
This is an old method of using Google Picker I believe @mifi. There is a new picker API for Google Photos and I believe it'll have to be a separate implementation: https://developers.google.com/photos/picker/guides/get-started-picker
At least how it appears to work is by making a POST request to https://photospicker.googleapis.com/v1/sessions which returns a few things including a pickerUri which can then the user can select photos in another tab/window using that URI from their Google Photos library for all photos and albums: https://developers.google.com/photos/picker/reference/rest/v1/sessions/create
However, I don't think Google allows you to access shared albums, etc. anymore at all unfortunately. So it just has to be something the user has in their library or album directly.
There was a problem hiding this comment.
yup I just found out about it too, i've done some testing and it seems to be working. It's a pity that shared albums don't work, but what can we do
There was a problem hiding this comment.
Awesome, it is hard to track down which APIs Google wants you to use haha. Agreed about shared albums, but exactly it is what it is.
- split into two plugins - implement photos picker - auto login - save access token in local storage - document - handle photos/files picked and send to companion - add new hook useStore for making it easier to use localStorage data in react - add new hook useUppyState for making it easier to use uppy state from react - add new hook useUppyPluginState for making it easier to plugin state from react - fix css error
which occurs in dev when js has been built before build:ts gets called
|
I think this is ready for review. The remaining points can be done iteratively in future PRs |
Murderlon
reviewed
Nov 20, 2024
There was a problem hiding this comment.
Lot of hard work to get this, well done!
Initial review below, still need to test it locally. Two Qs:
- Can you add the google account setup to our [email protected] account? Everyone already has those credentials setup so that would be great.
- If you use both google drive and google photos, do we get duplicate requests for the scripts you're loading?
|
|
||
| import type { AsyncStore } from './Uppy' | ||
|
|
||
| export default function useStore( |
There was a problem hiding this comment.
Why is this in core? And If you want to sync a store into preact/react state that's what useSyncExternalStore is for?
| import type { State, Uppy } from './Uppy' | ||
|
|
||
| // todo merge with @uppy/react? | ||
| export function useUppyState< |
There was a problem hiding this comment.
I don't think this should be in core either. And we should try to avoid writing the exact same hooks from @uppy/react again.
| }, [accessToken, clientId, pickerType, setAccessToken, uppy]) | ||
|
|
||
| useEffect(() => { | ||
| ;(async () => { |
There was a problem hiding this comment.
I think we should try to keep as much async work out of the React render cycle as possible. It's tricky because of cancellation during re-renders, loading, error, and retrying that has to be manually synced with state and you just keep fighting useEffect, which become fragile if they start to depend on too many things.
This comment is applicable to a lot of code in this file.
It would be great if we can move as much as possible into the plugin class and keep the Preact UI simple, ideally also preventing us from duplicating new hooks.
StrixOSG
reviewed
Nov 21, 2024
There was a problem hiding this comment.
Just to add on to @Murderlon's review. I was testing locally here and I seem to get a 403 after maybe about an hour or so for Google Drive Picker. I got everything working properly except that once the access token expires after about that hour I have to go into my local storage and clear uppy:google-drive-picker:accessToken manually otherwise it doesn't allow me to even logout and sign back in. I've attached a screenshot here of the error but it looks like it doesn't even hit the API, it's just a 403 for Google Picker account selection according to the Network request. So it might need an expiry date attached so it can be checked on the client side? Otherwise I don't know how this could be refreshed.
An aside and maybe it's just my UI or CSS config but the login, pick files, logout buttons do not look great as they're just plain buttons. Thank you for your hard work on this @mifi, it's much appreciated.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Initial POC of Google Picker.
closes #5382
closes #5469
How to test:
Google Developer console setup
Instructions for enabling the Picker API in the Google Developer console for the first time:
apiKeyclientIdappIdNow set
VITE_GOOGLE_PICKER_CLIENT_ID,VITE_GOOGLE_PICKER_API_KEY,VITE_GOOGLE_PICKER_APP_IDin your.envfile and runyarn dev:with-companionProblems/discussion
redirect_urineeds to be frontend URI, and wilcard is not allowed. Update: I've madeappId,clientIdandapiKeyas Uppy options (not Companion) so this should be OK because people can then use their own credentials.Cannot seem to get google photos to work (only Google Drive)TODO
tsconfig.json(I have just copy-pasted from google photos)tsconfig.build.json(I have just copy-pasted from google photos)google-picker.mdx(I have just copy-pasted from google photos)google-picker/README.md(I have just copy-pasted from google photos)onPickedadd the files along with any metadata needed by companion to download the files, like accessToken, clientId etc.clientId,scope,accessToken. The endpoint will callgapi.client.drive.files.get, and stream (download/upload) the file to the destination (e.g. tus/s3 etc). Need to make sure the endpoint is secure somehow in a way that it cannot be abused/exploited.this.provideris not needed in GooglePicker.tssetSelectFolderEnabled(true))