@@ -389,13 +402,9 @@ export default function AddData() {
ListenBrainz-SMP
- , a{" "}
-
- Foobar2000
- {" "}
- plugin for submitting and retrieving playlists from ListenBrainz (+
- Spotify), as well as retrieving recommendations or submitting feedback
- on tracks.
+ , a Foobar2000 plugin for submitting and retrieving playlists from
+ ListenBrainz (+ Spotify). Can retrieve recommendations and submit
+ track feedback.
@@ -407,9 +416,17 @@ export default function AddData() {
Foobar2000{" "}
- plugin for syncing local playlists (in multiple formats) with
- ListenBrainz (+ Spotify). Tracks playlists changes and also allows to
- resolve tracks with local content and YouTube links.
+ plugin for syncing local playlists with ListenBrainz (+ Spotify).
+ Tracks playlists changes and resolves tracks with local content and
+ YouTube links.
+
@@ -437,15 +454,19 @@ export default function AddData() {
- Wrapped-SMP
+ BrainzBot
- , a{" "}
+ , a Discord bot that can be user-run or added to your server. Showcase
+ what you're listening to, share charts, album grids, tag clouds,
+ and more.
+
+
- Foobar2000
- {" "}
- plugin for creating reports based on user listens similar to the one
- found at Spotify. Suggested playlists use ListenBrainz recommendations
- (without requiring listens upload to the server).
+ Wrapped-SMP
+
+ , a Foobar2000 plugin that creates listening reports, similar to
+ Spotify's annual report. Can utilize ListenBrainz
+ recommendations.
diff --git a/frontend/js/src/album/utils.tsx b/frontend/js/src/album/utils.tsx
index dddb43a193..3c6904ff20 100644
--- a/frontend/js/src/album/utils.tsx
+++ b/frontend/js/src/album/utils.tsx
@@ -69,6 +69,7 @@ export type ListeningStats = {
export function getRelIconLink(relName: string, relValue: string) {
let icon;
let color;
+ let isYoutube = false;
switch (relName) {
case "streaming":
case "free streaming":
@@ -84,6 +85,7 @@ export function getRelIconLink(relName: string, relValue: string) {
case "youtube music":
icon = faYoutube;
color = dataSourcesInfo.youtube.color;
+ isYoutube = true;
break;
case "soundcloud":
icon = faSoundcloud;
@@ -130,6 +132,13 @@ export function getRelIconLink(relName: string, relValue: string) {
icon = faLink;
break;
}
+ let style = {};
+ if (isYoutube) {
+ // Youtube forces us to follow their branding guidelines to the letter,
+ // so we need to force a minimum height of 20px for the icon path inside the svg
+ // [poo emoji]
+ style = { height: "26.7px", width: "auto" };
+ }
return (
-
+
);
}
diff --git a/frontend/js/src/artist/ArtistPage.tsx b/frontend/js/src/artist/ArtistPage.tsx
index dc1bc74170..1362639dba 100644
--- a/frontend/js/src/artist/ArtistPage.tsx
+++ b/frontend/js/src/artist/ArtistPage.tsx
@@ -72,14 +72,15 @@ function SortingButtons({
);
}
-interface ReleaseGroupWithSecondaryTypes extends ReleaseGroup {
+interface ReleaseGroupWithSecondaryTypesAndListenCount extends ReleaseGroup {
secondary_types: string[];
+ total_listen_count: number | null;
}
export type ArtistPageProps = {
popularRecordings: PopularRecording[];
artist: MusicBrainzArtist;
- releaseGroups: ReleaseGroupWithSecondaryTypes[];
+ releaseGroups: ReleaseGroupWithSecondaryTypesAndListenCount[];
similarArtists: {
artists: SimilarArtist[];
topReleaseGroupColor: ReleaseColor | undefined;
@@ -140,13 +141,17 @@ export default function ArtistPage(): JSX.Element {
);
const sortReleaseGroups = (
- releaseGroupsInput: ReleaseGroupWithSecondaryTypes[]
+ releaseGroupsInput: ReleaseGroupWithSecondaryTypesAndListenCount[]
) =>
orderBy(
releaseGroupsInput,
[
- sort === "release_date" ? (rg) => rg.date || "" : "total_listen_count",
- sort === "release_date" ? "total_listen_count" : (rg) => rg.date || "",
+ sort === "release_date"
+ ? (rg) => rg.date || ""
+ : (rg) => rg.total_listen_count ?? 0,
+ sort === "release_date"
+ ? (rg) => rg.total_listen_count ?? 0
+ : (rg) => rg.date || "",
"name",
],
["desc", "desc", "asc"]
@@ -168,7 +173,7 @@ export default function ArtistPage(): JSX.Element {
const groupedReleaseGroups: Record<
string,
- ReleaseGroupWithSecondaryTypes[]
+ ReleaseGroupWithSecondaryTypesAndListenCount[]
> = {};
sortedRgGroupsKeys.forEach((type) => {
groupedReleaseGroups[type] = sortReleaseGroups(rgGroups[type]);
@@ -276,6 +281,16 @@ export default function ArtistPage(): JSX.Element {
const releaseGroupTypesNames = Object.entries(groupedReleaseGroups);
+ // Only show "full discography" button if there are more than 4 rows
+ // in total across categories, after which we crop the container
+ const showFullDiscographyButton =
+ releaseGroupTypesNames.reduce(
+ (rows, curr) =>
+ // add up the number of rows (max of 2 rows in the css grid)
+ rows + (curr[1].length > COVER_ART_SINGLE_ROW_COUNT ? 2 : 1),
+ 0
+ ) > 4;
+
return (
@@ -506,7 +521,11 @@ export default function ArtistPage(): JSX.Element {
- The Last.fm importer only checks for new Last.fm listens, since your
- last Last.fm import. To do this, it stores the timestamp of the most
- recent listen submitted in your last import. Resetting the timestamp
- will make the next import check your entire Last.fm history. This will
- not create duplicates in your ListenBrainz listen history.
-
-
-
- );
-}
diff --git a/frontend/js/src/settings/routes/index.tsx b/frontend/js/src/settings/routes/index.tsx
index b2944ddbd9..107dedd80e 100644
--- a/frontend/js/src/settings/routes/index.tsx
+++ b/frontend/js/src/settings/routes/index.tsx
@@ -54,15 +54,6 @@ const getSettingsRoutes = (): RouteObject[] => {
return { Component: Import.default };
},
},
- {
- path: "resetlatestimportts/",
- lazy: async () => {
- const ResetImportTimestamp = await import(
- "../resetlatestimportts/ResetLatestImports"
- );
- return { Component: ResetImportTimestamp.default };
- },
- },
{
path: "missing-data/",
loader: RouteQueryLoader("missing-data"),
diff --git a/frontend/js/src/settings/routes/redirectRoutes.tsx b/frontend/js/src/settings/routes/redirectRoutes.tsx
index 15f099e39e..f745c50a0a 100644
--- a/frontend/js/src/settings/routes/redirectRoutes.tsx
+++ b/frontend/js/src/settings/routes/redirectRoutes.tsx
@@ -24,10 +24,6 @@ const getRedirectRoutes = (): RouteObject[] => {
path: "import/",
element: ,
},
- {
- path: "resetlatestimportts/",
- element: ,
- },
{
path: "missing-data/",
element: ,
diff --git a/frontend/js/tests/common/brainzplayer/BrainzPlayer.test.tsx b/frontend/js/tests/common/brainzplayer/BrainzPlayer.test.tsx
index 363327cfb7..b6028897f7 100644
--- a/frontend/js/tests/common/brainzplayer/BrainzPlayer.test.tsx
+++ b/frontend/js/tests/common/brainzplayer/BrainzPlayer.test.tsx
@@ -94,6 +94,13 @@ function BrainzPlayerWithWrapper(brainzPlayerProps: {
const mockDispatch = jest.fn();
+jest.mock("react-router-dom", () => ({
+ ...jest.requireActual("react-router-dom"),
+ useLocation: () => ({
+ pathname: "/user/foobar/",
+ }),
+}));
+
describe("BrainzPlayer", () => {
beforeEach(() => {
(useBrainzPlayerContext as jest.MockedFunction<
diff --git a/frontend/js/tests/lastfm/LastFMImporter.test.tsx b/frontend/js/tests/lastfm/LibreFMImporter.test.tsx
similarity index 81%
rename from frontend/js/tests/lastfm/LastFMImporter.test.tsx
rename to frontend/js/tests/lastfm/LibreFMImporter.test.tsx
index 76e56b422a..3e9fc6182d 100644
--- a/frontend/js/tests/lastfm/LastFMImporter.test.tsx
+++ b/frontend/js/tests/lastfm/LibreFMImporter.test.tsx
@@ -1,12 +1,8 @@
import * as React from "react";
-import { mount, ReactWrapper, shallow, ShallowWrapper } from "enzyme";
+import { mount, shallow } from "enzyme";
import fetchMock from "jest-fetch-mock";
import { act } from "react-dom/test-utils";
-import LastFmImporter, {
- LASTFM_RETRIES,
- ImporterState,
- ImporterProps,
-} from "../../src/lastfm/LastFMImporter";
+import LibreFmImporter, { RETRIES } from "../../src/lastfm/LibreFMImporter";
// Mock data to test functions
import * as page from "../__mocks__/page.json";
import * as getInfo from "../__mocks__/getInfo.json";
@@ -14,9 +10,8 @@ import * as getInfoNoPlayCount from "../__mocks__/getInfoNoPlayCount.json";
// Output for the mock data
import * as encodeScrobbleOutput from "../__mocks__/encodeScrobbleOutput.json";
import * as lastFMPrivateUser from "../__mocks__/lastFMPrivateUser.json";
-import { waitForComponentToPaint } from "../test-utils";
-jest.useFakeTimers({advanceTimers: true});
+jest.useFakeTimers({ advanceTimers: true });
const props = {
user: {
id: "id",
@@ -25,46 +20,43 @@ const props = {
},
profileUrl: "http://profile",
apiUrl: "apiUrl",
- lastfmApiUrl: "http://ws.audioscrobbler.com/2.0/",
- lastfmApiKey: "foobar",
librefmApiUrl: "http://libre.fm/2.0/",
librefmApiKey: "barfoo",
};
-describe("LastFMImporter", () => {
+describe("LibreFMImporter", () => {
describe("encodeScrobbles", () => {
it("encodes the given scrobbles correctly", () => {
- expect(LastFmImporter.encodeScrobbles(page)).toEqual(
+ expect(LibreFmImporter.encodeScrobbles(page)).toEqual(
encodeScrobbleOutput
);
});
});
- let instance: LastFmImporter;
+ let instance: LibreFmImporter;
describe("getNumberOfPages", () => {
- beforeAll(()=>{
+ beforeAll(() => {
fetchMock.enableMocks();
// Mock function for fetch
fetchMock.mockResponse(JSON.stringify(page));
- })
+ });
beforeEach(() => {
- const wrapper = shallow();
+ const wrapper = shallow();
instance = wrapper.instance();
- instance.setState({ lastfmUsername: "dummyUser" });
+ instance.setState({ librefmUsername: "dummyUser" });
});
it("should call with the correct url", async () => {
instance.getNumberOfPages();
expect(fetchMock).toHaveBeenCalledWith(
- `${props.lastfmApiUrl}?method=user.getrecenttracks&user=${instance.state.lastfmUsername}&api_key=${props.lastfmApiKey}&from=1&format=json`
+ `${props.librefmApiUrl}?method=user.getrecenttracks&user=${instance.state.librefmUsername}&api_key=${props.librefmApiKey}&from=1&format=json`
);
const num = await instance.getNumberOfPages();
expect(num).toBe(1);
});
-
it("should return -1 if there is an error", async () => {
// Mock function for failed fetch
window.fetch = jest.fn().mockImplementation(() => {
@@ -80,9 +72,9 @@ describe("LastFMImporter", () => {
describe("getTotalNumberOfScrobbles", () => {
beforeEach(() => {
- const wrapper = shallow();
+ const wrapper = shallow();
instance = wrapper.instance();
- instance.setState({ lastfmUsername: "dummyUser" });
+ instance.setState({ librefmUsername: "dummyUser" });
// Mock function for fetch
window.fetch = jest.fn().mockImplementation(() => {
return Promise.resolve({
@@ -96,7 +88,7 @@ describe("LastFMImporter", () => {
instance.getTotalNumberOfScrobbles();
expect(window.fetch).toHaveBeenCalledWith(
- `${props.lastfmApiUrl}?method=user.getinfo&user=${instance.state.lastfmUsername}&api_key=${props.lastfmApiKey}&format=json`
+ `${props.librefmApiUrl}?method=user.getinfo&user=${instance.state.librefmUsername}&api_key=${props.librefmApiKey}&format=json`
);
});
@@ -146,7 +138,9 @@ describe("LastFMImporter", () => {
json: () => Promise.resolve({ message: "User not found", error: 6 }),
})
);
- instance.setState({ lastfmUsername: "nonexistentusernamedonttryathome" });
+ instance.setState({
+ librefmUsername: "nonexistentusernamedonttryathome",
+ });
await expect(instance.getTotalNumberOfScrobbles()).rejects.toThrowError(
"User not found"
@@ -172,9 +166,9 @@ describe("LastFMImporter", () => {
});
beforeEach(() => {
- const wrapper = shallow();
+ const wrapper = shallow();
instance = wrapper.instance();
- instance.setState({ lastfmUsername: "dummyUser" });
+ instance.setState({ librefmUsername: "dummyUser" });
// Mock function for fetch
window.fetch = jest.fn().mockImplementation(() => {
return Promise.resolve({
@@ -185,10 +179,10 @@ describe("LastFMImporter", () => {
});
it("should call with the correct url", () => {
- instance.getPage(1, LASTFM_RETRIES);
+ instance.getPage(1, RETRIES);
expect(window.fetch).toHaveBeenCalledWith(
- `${props.lastfmApiUrl}?method=user.getrecenttracks&user=${instance.state.lastfmUsername}&api_key=${props.lastfmApiKey}&from=1&page=1&format=json`
+ `${props.librefmApiUrl}?method=user.getrecenttracks&user=${instance.state.librefmUsername}&api_key=${props.librefmApiKey}&from=1&page=1&format=json`
);
});
@@ -204,15 +198,15 @@ describe("LastFMImporter", () => {
const getPageSpy = jest.spyOn(instance, "getPage");
let finalValue;
try {
- finalValue = await instance.getPage(1, LASTFM_RETRIES);
+ finalValue = await instance.getPage(1, RETRIES);
} catch (err) {
- expect(getPageSpy).toHaveBeenCalledTimes(1 + LASTFM_RETRIES);
+ expect(getPageSpy).toHaveBeenCalledTimes(1 + RETRIES);
expect(finalValue).toBeUndefined();
// This error message is also displayed to the user
expect(err).toEqual(
new Error(
- `Failed to fetch page 1 from lastfm after ${LASTFM_RETRIES} retries: Error: Status 503`
+ `Failed to fetch page 1 from librefm after ${RETRIES} retries: Error: Status 503`
)
);
}
@@ -242,7 +236,7 @@ describe("LastFMImporter", () => {
});
const getPageSpy = jest.spyOn(instance, "getPage");
- const finalValue = await instance.getPage(1, LASTFM_RETRIES);
+ const finalValue = await instance.getPage(1, RETRIES);
expect(getPageSpy).toHaveBeenCalledTimes(3);
expect(finalValue).toEqual(encodeScrobbleOutput);
@@ -257,7 +251,7 @@ describe("LastFMImporter", () => {
});
});
const getPageSpy = jest.spyOn(instance, "getPage");
- const finalValue = await instance.getPage(1, LASTFM_RETRIES);
+ const finalValue = await instance.getPage(1, RETRIES);
expect(getPageSpy).toHaveBeenCalledTimes(1);
expect(finalValue).toEqual(undefined);
@@ -272,7 +266,7 @@ describe("LastFMImporter", () => {
});
});
const getPageSpy = jest.spyOn(instance, "getPage");
- const finalValue = await instance.getPage(1, LASTFM_RETRIES);
+ const finalValue = await instance.getPage(1, RETRIES);
expect(getPageSpy).toHaveBeenCalledTimes(1);
expect(finalValue).toEqual(undefined);
@@ -296,7 +290,7 @@ describe("LastFMImporter", () => {
});
const getPageSpy = jest.spyOn(instance, "getPage");
- const finalValue = await instance.getPage(1, LASTFM_RETRIES);
+ const finalValue = await instance.getPage(1, RETRIES);
expect(getPageSpy).toHaveBeenCalledTimes(2);
expect(finalValue).toEqual(encodeScrobbleOutput);
@@ -304,19 +298,19 @@ describe("LastFMImporter", () => {
it("should call encodeScrobbles", async () => {
// Mock function for encodeScrobbles
- LastFmImporter.encodeScrobbles = jest.fn(() => ["foo", "bar"]);
+ LibreFmImporter.encodeScrobbles = jest.fn(() => ["foo", "bar"]);
- const data = await instance.getPage(1, LASTFM_RETRIES);
- expect(LastFmImporter.encodeScrobbles).toHaveBeenCalledTimes(1);
+ const data = await instance.getPage(1, RETRIES);
+ expect(LibreFmImporter.encodeScrobbles).toHaveBeenCalledTimes(1);
expect(data).toEqual(["foo", "bar"]);
});
});
describe("submitPage", () => {
beforeEach(() => {
- const wrapper = shallow();
+ const wrapper = shallow();
instance = wrapper.instance();
- instance.setState({ lastfmUsername: "dummyUser" });
+ instance.setState({ librefmUsername: "dummyUser" });
instance.getRateLimitDelay = jest.fn().mockImplementation(() => 0);
instance.APIService.submitListens = jest.fn().mockImplementation(() => {
return Promise.resolve({
@@ -394,9 +388,9 @@ describe("LastFMImporter", () => {
describe("getUserPrivacy", () => {
beforeEach(() => {
- const wrapper = shallow();
+ const wrapper = shallow();
instance = wrapper.instance();
- instance.setState({ lastfmUsername: "dummyUser" });
+ instance.setState({ librefmUsername: "dummyUser" });
// Needed for startImport
instance.APIService.getLatestImport = jest.fn().mockImplementation(() => {
@@ -414,7 +408,7 @@ describe("LastFMImporter", () => {
instance.getUserPrivacy();
expect(window.fetch).toHaveBeenCalledWith(
- `${props.lastfmApiUrl}?method=user.getrecenttracks&user=${instance.state.lastfmUsername}&api_key=${props.lastfmApiKey}&format=json`
+ `${props.librefmApiUrl}?method=user.getrecenttracks&user=${instance.state.librefmUsername}&api_key=${props.librefmApiKey}&format=json`
);
});
@@ -472,30 +466,30 @@ describe("LastFMImporter", () => {
});
});
- describe("LastFmImporter Page", () => {
+ describe("LibreFmImporter Page", () => {
it("renders", () => {
- const wrapper = mount();
- expect(wrapper.getDOMNode()).toHaveTextContent("Choose a service");
- expect(wrapper.getDOMNode()).toHaveTextContent("Last.fm");
- expect(wrapper.getDOMNode()).toHaveTextContent("Libre.fm");
+ const wrapper = mount();
+ expect(wrapper.getDOMNode()).toHaveTextContent(
+ "Your librefm username:Import listens"
+ );
});
it("modal renders when button clicked", () => {
- const wrapper = shallow();
+ const wrapper = shallow();
// Simulate submiting the form
wrapper.find("form").simulate("submit", {
preventDefault: () => null,
});
// Test if the show property has been set to true
- expect(wrapper.exists("LastFMImporterModal")).toBe(true);
+ expect(wrapper.exists("LibreFMImporterModal")).toBe(true);
});
it("submit button is disabled when input is empty", () => {
- const wrapper = shallow();
+ const wrapper = shallow();
act(() => {
// Make sure that the input is empty
- wrapper.setState({ lastfmUsername: "" });
+ wrapper.setState({ librefmUsername: "" });
});
// Test if button is disabled
@@ -504,7 +498,7 @@ describe("LastFMImporter", () => {
it("should properly convert latest imported timestamp to string", () => {
// Check getlastImportedString() and formatting
- const data = LastFmImporter.encodeScrobbles(page);
+ const data = LibreFmImporter.encodeScrobbles(page);
const lastImportedDate = new Date(data[0].listened_at * 1000);
const msg = lastImportedDate.toLocaleString("en-US", {
month: "short",
@@ -515,16 +509,18 @@ describe("LastFMImporter", () => {
hour12: true,
});
- expect(LastFmImporter.getlastImportedString(data[0])).toMatch(msg);
- expect(LastFmImporter.getlastImportedString(data[0])).not.toHaveLength(0);
+ expect(LibreFmImporter.getlastImportedString(data[0])).toMatch(msg);
+ expect(LibreFmImporter.getlastImportedString(data[0])).not.toHaveLength(
+ 0
+ );
});
});
describe("importLoop", () => {
beforeEach(() => {
- const wrapper = shallow();
+ const wrapper = shallow();
instance = wrapper.instance();
- instance.setState({ lastfmUsername: "dummyUser" });
+ instance.setState({ librefmUsername: "dummyUser" });
// needed for startImport
instance.APIService.getLatestImport = jest.fn().mockImplementation(() => {
return Promise.resolve(0);
diff --git a/frontend/js/tests/lastfm/LastFMImporterModal.test.tsx b/frontend/js/tests/lastfm/LibreFMImporterModal.test.tsx
similarity index 77%
rename from frontend/js/tests/lastfm/LastFMImporterModal.test.tsx
rename to frontend/js/tests/lastfm/LibreFMImporterModal.test.tsx
index 140e36c760..7af298b966 100644
--- a/frontend/js/tests/lastfm/LastFMImporterModal.test.tsx
+++ b/frontend/js/tests/lastfm/LibreFMImporterModal.test.tsx
@@ -2,7 +2,7 @@ import * as React from "react";
import { mount, shallow } from "enzyme";
import { act } from "react-dom/test-utils";
-import LastFMImporterModal from "../../src/lastfm/LastFMImporterModal";
+import LibreFMImporterModal from "../../src/lastfm/LibreFMImporterModal";
const props = {
disable: false,
@@ -10,15 +10,15 @@ const props = {
onClose: (event: React.MouseEvent) => {},
};
-describe("LastFmImporterModal", () => {
+describe("LibreFMImporterModal", () => {
it("renders", () => {
- const wrapper = mount();
+ const wrapper = mount();
expect(wrapper.find("#listen-progress-container")).toHaveLength(1);
expect(wrapper.find("button")).toHaveLength(1);
});
it("close button is disabled/enabled based upon props", async () => {
- const wrapper = shallow();
+ const wrapper = shallow();
// Test if close button is disabled
await act(() => {
wrapper.setProps({ disable: true });
diff --git a/listenbrainz/db/external_service_oauth.py b/listenbrainz/db/external_service_oauth.py
index b148165015..370c8782c4 100644
--- a/listenbrainz/db/external_service_oauth.py
+++ b/listenbrainz/db/external_service_oauth.py
@@ -1,3 +1,4 @@
+from datetime import datetime
from typing import List, Optional, Union
from sqlalchemy import text
@@ -7,8 +8,9 @@
import sqlalchemy
-def save_token(db_conn, user_id: int, service: ExternalServiceType, access_token: str, refresh_token: Optional[str],
- token_expires_ts: int, record_listens: bool, scopes: List[str], external_user_id: Optional[str] = None):
+def save_token(db_conn, user_id: int, service: ExternalServiceType, access_token: Optional[str], refresh_token: Optional[str],
+ token_expires_ts: Optional[int], record_listens: bool, scopes: Optional[List[str]], external_user_id: Optional[str] = None,
+ latest_listened_at: Optional[datetime] = None):
""" Add a row to the external_service_oauth table for specified user with corresponding tokens and information.
Args:
@@ -21,6 +23,7 @@ def save_token(db_conn, user_id: int, service: ExternalServiceType, access_token
record_listens: True if user wishes to import listens, False otherwise
scopes: the oauth scopes
external_user_id: the user's id in the external linked service
+ latest_listened_at: last listen import time
"""
# regardless of whether a row is inserted or updated, the end result of the query
# should remain the same. if not so, weird things can happen as it is likely we
@@ -29,7 +32,7 @@ def save_token(db_conn, user_id: int, service: ExternalServiceType, access_token
# to use the new values. any column which does not have a new value to be set should
# be explicitly set to the default value (which would have been used if the row was
# inserted instead).
- token_expires = utils.unix_timestamp_to_datetime(token_expires_ts)
+ token_expires = utils.unix_timestamp_to_datetime(token_expires_ts) if token_expires_ts else None
result = db_conn.execute(sqlalchemy.text("""
INSERT INTO external_service_oauth AS eso
(user_id, external_user_id, service, access_token, refresh_token, token_expires, scopes)
@@ -58,20 +61,21 @@ def save_token(db_conn, user_id: int, service: ExternalServiceType, access_token
external_service_oauth_id = result.fetchone().id
db_conn.execute(sqlalchemy.text("""
INSERT INTO listens_importer
- (external_service_oauth_id, user_id, service)
+ (external_service_oauth_id, user_id, service, latest_listened_at)
VALUES
- (:external_service_oauth_id, :user_id, :service)
+ (:external_service_oauth_id, :user_id, :service, :latest_listened_at)
ON CONFLICT (user_id, service) DO UPDATE SET
external_service_oauth_id = EXCLUDED.external_service_oauth_id,
user_id = EXCLUDED.user_id,
service = EXCLUDED.service,
last_updated = NULL,
- latest_listened_at = NULL,
+ latest_listened_at = EXCLUDED.latest_listened_at,
error_message = NULL
"""), {
"external_service_oauth_id": external_service_oauth_id,
"user_id": user_id,
- "service": service.value
+ "service": service.value,
+ "latest_listened_at": latest_listened_at,
})
db_conn.commit()
@@ -157,20 +161,24 @@ def get_token(db_conn, user_id: int, service: ExternalServiceType) -> Union[dict
service: the service for which the token should be fetched
"""
result = db_conn.execute(sqlalchemy.text("""
- SELECT user_id
+ SELECT "user".id AS user_id
, "user".musicbrainz_id
, "user".musicbrainz_row_id
- , service
+ , eso.service
, access_token
, refresh_token
- , last_updated
+ , eso.last_updated
, token_expires
, scopes
, external_user_id
- FROM external_service_oauth
+ , li.latest_listened_at
+ FROM external_service_oauth eso
JOIN "user"
- ON "user".id = external_service_oauth.user_id
- WHERE user_id = :user_id AND service = :service
+ ON "user".id = eso.user_id
+ LEFT JOIN listens_importer li
+ ON li.external_service_oauth_id = eso.id
+ WHERE "user".id = :user_id
+ AND eso.service = :service
"""), {
'user_id': user_id,
'service': service.value
diff --git a/listenbrainz/db/listens_importer.py b/listenbrainz/db/listens_importer.py
index fd2f80e990..b76732f6d5 100644
--- a/listenbrainz/db/listens_importer.py
+++ b/listenbrainz/db/listens_importer.py
@@ -1,6 +1,8 @@
import datetime
from typing import Optional, Union
+from sqlalchemy import text
+
from data.model.external_service import ExternalServiceType
from listenbrainz import utils
import sqlalchemy
@@ -76,3 +78,36 @@ def get_latest_listened_at(db_conn, user_id: int, service: ExternalServiceType)
})
row = result.fetchone()
return row.latest_listened_at if row else None
+
+
+def get_active_users_to_process(db_conn, service, exclude_error=False) -> list[dict]:
+ """ Returns a list of users whose listens should be imported from the external service.
+ """
+ filters = ["external_service_oauth.service = :service"]
+ if exclude_error:
+ filters.append("error_message IS NULL")
+ filter_str = " AND ".join(filters)
+
+ result = db_conn.execute(text(f"""
+ SELECT external_service_oauth.user_id
+ , "user".musicbrainz_id
+ , "user".musicbrainz_row_id
+ , access_token
+ , refresh_token
+ , listens_importer.last_updated
+ , token_expires
+ , scopes
+ , latest_listened_at
+ , external_service_oauth.external_user_id
+ , error_message
+ FROM external_service_oauth
+ JOIN "user"
+ ON "user".id = external_service_oauth.user_id
+ JOIN listens_importer
+ ON listens_importer.external_service_oauth_id = external_service_oauth.id
+ WHERE {filter_str}
+ ORDER BY latest_listened_at DESC NULLS LAST
+ """), {"service": service.value})
+ users = [row for row in result.mappings()]
+ db_conn.rollback()
+ return users
diff --git a/listenbrainz/db/spotify.py b/listenbrainz/db/spotify.py
index b7b25b9124..e86b7fc178 100644
--- a/listenbrainz/db/spotify.py
+++ b/listenbrainz/db/spotify.py
@@ -1,35 +1,8 @@
from typing import List, Optional
-from listenbrainz import db
import sqlalchemy
-def get_active_users_to_process(db_conn) -> List[dict]:
- """ Returns a list of users whose listens should be imported from Spotify.
- """
- result = db_conn.execute(sqlalchemy.text("""
- SELECT external_service_oauth.user_id
- , "user".musicbrainz_id
- , "user".musicbrainz_row_id
- , access_token
- , refresh_token
- , listens_importer.last_updated
- , token_expires
- , scopes
- , latest_listened_at
- , error_message
- FROM external_service_oauth
- JOIN "user"
- ON "user".id = external_service_oauth.user_id
- JOIN listens_importer
- ON listens_importer.external_service_oauth_id = external_service_oauth.id
- WHERE external_service_oauth.service = 'spotify'
- AND error_message IS NULL
- ORDER BY latest_listened_at DESC NULLS LAST
- """))
- return [row for row in result.mappings()]
-
-
def get_user_import_details(db_conn, user_id: int) -> Optional[dict]:
""" Return user's spotify linking details to display on connect services page
diff --git a/listenbrainz/db/tests/test_spotify.py b/listenbrainz/db/tests/test_spotify.py
index 98b46c537b..006a65a559 100644
--- a/listenbrainz/db/tests/test_spotify.py
+++ b/listenbrainz/db/tests/test_spotify.py
@@ -37,7 +37,7 @@ def test_get_active_users_to_process(self):
record_listens=True,
scopes=['user-read-recently-played']
)
- users = db_spotify.get_active_users_to_process(self.db_conn)
+ users = db_import.get_active_users_to_process(self.db_conn, ExternalServiceType.SPOTIFY)
self.assertEqual(len(users), 2)
self.assertEqual(users[0]['user_id'], self.user['id'])
self.assertEqual(users[0]['musicbrainz_row_id'], 1)
@@ -59,7 +59,7 @@ def test_get_active_users_to_process(self):
t = int(time.time())
db_import.update_latest_listened_at(self.db_conn, user2['id'], ExternalServiceType.SPOTIFY, t + 20)
db_import.update_latest_listened_at(self.db_conn, self.user['id'], ExternalServiceType.SPOTIFY, t + 10)
- users = db_spotify.get_active_users_to_process(self.db_conn)
+ users = db_import.get_active_users_to_process(self.db_conn, ExternalServiceType.SPOTIFY)
self.assertEqual(len(users), 3)
self.assertEqual(users[0]['user_id'], user2['id'])
self.assertEqual(users[1]['user_id'], self.user['id'])
@@ -67,7 +67,7 @@ def test_get_active_users_to_process(self):
db_import.update_import_status(self.db_conn, user2['id'], ExternalServiceType.SPOTIFY, 'something broke')
db_import.update_import_status(self.db_conn, user3['id'], ExternalServiceType.SPOTIFY, 'oops.')
- users = db_spotify.get_active_users_to_process(self.db_conn)
+ users = db_import.get_active_users_to_process(self.db_conn, ExternalServiceType.SPOTIFY, True)
self.assertEqual(len(users), 1)
self.assertEqual(users[0]['user_id'], self.user['id'])
diff --git a/listenbrainz/domain/importer_service.py b/listenbrainz/domain/importer_service.py
index d0767857e2..00775d8ca2 100644
--- a/listenbrainz/domain/importer_service.py
+++ b/listenbrainz/domain/importer_service.py
@@ -1,5 +1,7 @@
from abc import ABC
-from typing import List, Union
+from typing import Union
+
+from flask import current_app
from listenbrainz.domain.external_service import ExternalService, ExternalServiceError
from listenbrainz.db import listens_importer
@@ -10,9 +12,9 @@ class ImporterService(ExternalService, ABC):
""" Base class that external music services which also allow to import listen history
to ListenBrainz should implement."""
- def get_active_users_to_process(self) -> List[dict]:
+ def get_active_users_to_process(self, exclude_error=True) -> list[dict]:
""" Return list of active users for importing listens. """
- raise NotImplementedError()
+ return listens_importer.get_active_users_to_process(db_conn, self.service, exclude_error)
def update_user_import_status(self, user_id: int, error: str = None):
""" Update the last_update field for user with specified user ID.
diff --git a/listenbrainz/domain/lastfm.py b/listenbrainz/domain/lastfm.py
index 4add6040bf..ee37663d72 100644
--- a/listenbrainz/domain/lastfm.py
+++ b/listenbrainz/domain/lastfm.py
@@ -8,6 +8,9 @@
from brainzutils import musicbrainz_db
+from data.model.external_service import ExternalServiceType
+from listenbrainz.db import external_service_oauth
+from listenbrainz.domain.importer_service import ImporterService
from listenbrainz.webserver import db_conn
from listenbrainz.webserver.errors import APINotFound
@@ -138,3 +141,17 @@ def import_feedback(user_id: int, lfm_user: str) -> dict:
counts["inserted"] = len(recording_feedback)
return counts
+
+
+class LastfmService(ImporterService):
+
+ def __init__(self):
+ super(LastfmService, self).__init__(ExternalServiceType.LASTFM)
+
+ def add_new_user(self, user_id: int, token: dict) -> bool:
+ external_service_oauth.save_token(
+ db_conn, user_id=user_id, service=self.service, access_token=None, refresh_token=None,
+ token_expires_ts=None, record_listens=True, scopes=[], external_user_id=token["external_user_id"],
+ latest_listened_at=token["latest_listened_at"]
+ )
+ return True
diff --git a/listenbrainz/domain/spotify.py b/listenbrainz/domain/spotify.py
index 37bc5b6cea..4019941d62 100644
--- a/listenbrainz/domain/spotify.py
+++ b/listenbrainz/domain/spotify.py
@@ -194,8 +194,3 @@ def date_to_iso(date):
user['latest_listened_at_iso'] = date_to_iso(user['latest_listened_at'])
user['last_updated_iso'] = date_to_iso(user['last_updated'])
return user
-
- def get_active_users_to_process(self):
- """ Returns a list of Spotify user instances that need their Spotify listens imported.
- """
- return spotify.get_active_users_to_process(db_conn)
diff --git a/listenbrainz/spotify_updater/__init__.py b/listenbrainz/listens_importer/__init__.py
similarity index 100%
rename from listenbrainz/spotify_updater/__init__.py
rename to listenbrainz/listens_importer/__init__.py
diff --git a/listenbrainz/listens_importer/base.py b/listenbrainz/listens_importer/base.py
new file mode 100644
index 0000000000..caaab04a7f
--- /dev/null
+++ b/listenbrainz/listens_importer/base.py
@@ -0,0 +1,178 @@
+import abc
+import time
+from abc import abstractmethod
+
+from brainzutils import metrics
+from flask import current_app, render_template
+from psycopg2 import DatabaseError
+from sqlalchemy.exc import SQLAlchemyError
+from werkzeug.exceptions import InternalServerError, ServiceUnavailable
+
+from brainzutils.mail import send_mail
+
+from listenbrainz.db.exceptions import DatabaseException
+from listenbrainz.domain.external_service import ExternalServiceError
+from listenbrainz.webserver.errors import ListenValidationError
+from listenbrainz.webserver.models import SubmitListenUserMetadata
+from listenbrainz.webserver.views.api_tools import LISTEN_TYPE_IMPORT, insert_payload, validate_listen, \
+ LISTEN_TYPE_SINGLE, LISTEN_TYPE_PLAYING_NOW
+
+from listenbrainz.db import user as db_user
+import listenbrainz
+
+METRIC_UPDATE_INTERVAL = 60 # seconds
+
+
+class ListensImporter(abc.ABC):
+
+ def __init__(self, name, user_friendly_name, service):
+ self.name = name
+ self.user_friendly_name = user_friendly_name
+ self.service = service
+ # number of listens imported since last metric update was submitted
+ self._listens_imported_since_last_update = 0
+ self._metric_submission_time = time.monotonic() + METRIC_UPDATE_INTERVAL
+ self.exclude_error = True
+
+ def notify_error(self, musicbrainz_id: str, error: str):
+ """ Notifies specified user via email about error during Spotify import.
+
+ Args:
+ musicbrainz_id: the MusicBrainz ID of the user
+ error: a description of the error encountered.
+ """
+ user_email = db_user.get_by_mb_id(listenbrainz.webserver.db_conn, musicbrainz_id, fetch_email=True)["email"]
+ if not user_email:
+ return
+
+ link = current_app.config['SERVER_ROOT_URL'] + '/settings/music-services/details/'
+ text = render_template('emails/listens_importer_error.txt', error=error, link=link)
+ send_mail(
+ subject=f'ListenBrainz {self.user_friendly_name} Importer Error',
+ text=text,
+ recipients=[user_email],
+ from_name='ListenBrainz',
+ from_addr='noreply@' + current_app.config['MAIL_FROM_DOMAIN'],
+ )
+
+ def parse_and_validate_listen_items(self, converter, items):
+ """ Converts and validates the listens received from the external service API.
+
+ Args:
+ converter: a function to parse the incoming items that returns a tuple of (listen, listen_type)
+ items: a list of listen events received from the external
+
+ Returns:
+ tuple of (now playing listen, a list of recent listens to submit to ListenBrainz, timestamp of latest listen)
+ """
+ now_playing_listen = None
+ listens = []
+ latest_listen_ts = None
+
+ for item in items:
+ listen, listen_type = converter(item)
+
+ if listen_type == LISTEN_TYPE_IMPORT and \
+ (latest_listen_ts is None or listen['listened_at'] > latest_listen_ts):
+ latest_listen_ts = listen['listened_at']
+
+ try:
+ validate_listen(listen, listen_type)
+ if listen_type == LISTEN_TYPE_IMPORT or listen_type == LISTEN_TYPE_SINGLE:
+ listens.append(listen)
+
+ # set the first now playing listen to now_playing and ignore the rest
+ if listen_type == LISTEN_TYPE_PLAYING_NOW and now_playing_listen is None:
+ now_playing_listen = listen
+ except ListenValidationError:
+ pass
+ return now_playing_listen, listens, latest_listen_ts
+
+ def submit_listens_to_listenbrainz(self, user: dict, listens: list[dict], listen_type=LISTEN_TYPE_IMPORT):
+ """ Submit a batch of listens to ListenBrainz
+
+ Args:
+ user: the user whose listens are to be submitted, dict should contain
+ at least musicbrainz_id and user_id
+ listens: a list of listens to be submitted
+ listen_type: the type of listen (single, import, playing_now)
+ """
+ username = user['musicbrainz_id']
+ user_metadata = SubmitListenUserMetadata(user_id=user['user_id'], musicbrainz_id=username)
+ retries = 10
+ while retries >= 0:
+ try:
+ current_app.logger.debug('Submitting %d listens for user %s', len(listens), username)
+ insert_payload(listens, user_metadata, listen_type=listen_type)
+ current_app.logger.debug('Submitted!')
+ break
+ except (InternalServerError, ServiceUnavailable) as e:
+ retries -= 1
+ current_app.logger.error('ISE while trying to import listens for %s: %s', username, str(e))
+ if retries == 0:
+ raise ExternalServiceError('ISE while trying to import listens: %s', str(e))
+
+ @abstractmethod
+ def process_one_user(self, user):
+ pass
+
+ def process_all_users(self):
+ """ Get a batch of users to be processed and import their listens.
+
+ Returns:
+ (success, failure) where
+ success: the number of users whose plays were successfully imported.
+ failure: the number of users for whom we faced errors while importing.
+ """
+ try:
+ users = self.service.get_active_users_to_process(self.exclude_error)
+ except DatabaseException as e:
+ listenbrainz.webserver.db_conn.rollback()
+ current_app.logger.error('Cannot get list of users due to error %s', str(e), exc_info=True)
+ return 0, 0
+
+ if not users:
+ return 0, 0
+
+ current_app.logger.info('Process %d users...' % len(users))
+ success = 0
+ failure = 0
+ for user in users:
+ try:
+ self._listens_imported_since_last_update += self.process_one_user(user)
+ success += 1
+ except (DatabaseException, DatabaseError, SQLAlchemyError):
+ listenbrainz.webserver.db_conn.rollback()
+ current_app.logger.error(f'{self.name} could not import listens for user %s:',
+ user['musicbrainz_id'], exc_info=True)
+ except Exception:
+ current_app.logger.error(f'{self.name} could not import listens for user %s:',
+ user['musicbrainz_id'], exc_info=True)
+ failure += 1
+
+ if time.monotonic() > self._metric_submission_time:
+ self._metric_submission_time += METRIC_UPDATE_INTERVAL
+ metrics.set(self.name, imported_listens=self._listens_imported_since_last_update)
+ _listens_imported_since_last_update = 0
+
+ current_app.logger.info('Processed %d users successfully!', success)
+ current_app.logger.info('Encountered errors while processing %d users.', failure)
+ return success, failure
+
+ def main(self):
+ current_app.logger.info(f'{self.name} started...')
+ while True:
+ t = time.monotonic()
+ success, failure = self.process_all_users()
+ total_users = success + failure
+ if total_users > 0:
+ total_time = time.monotonic() - t
+ avg_time = total_time / total_users
+ metrics.set(self.name,
+ users_processed=total_users,
+ time_to_process_all_users=total_time,
+ time_to_process_one_user=avg_time)
+ current_app.logger.info('All %d users in batch have been processed.', total_users)
+ current_app.logger.info('Total time taken: %.2f s, average time per user: %.2f s.', total_time,
+ avg_time)
+ time.sleep(10)
diff --git a/listenbrainz/listens_importer/lastfm.py b/listenbrainz/listens_importer/lastfm.py
new file mode 100644
index 0000000000..d323434f55
--- /dev/null
+++ b/listenbrainz/listens_importer/lastfm.py
@@ -0,0 +1,162 @@
+#!/usr/bin/python3
+
+import requests
+from flask import current_app
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
+
+from listenbrainz.domain.external_service import ExternalServiceError, ExternalServiceAPIError
+from listenbrainz.domain.lastfm import LastfmService
+from listenbrainz.listens_importer.base import ListensImporter
+from listenbrainz.listenstore import LISTEN_MINIMUM_DATE
+from listenbrainz.webserver import create_app
+from listenbrainz.webserver.views.api_tools import LISTEN_TYPE_IMPORT, \
+ LISTEN_TYPE_PLAYING_NOW
+
+
+class LastfmImporter(ListensImporter):
+
+ def __init__(self):
+ super(LastfmImporter, self).__init__(
+ name='LastfmImporter',
+ user_friendly_name="Last.fm",
+ service=LastfmService(),
+ )
+
+ @staticmethod
+ def convert_scrobble_to_listen(scrobble):
+ """ Converts data retrieved from the last.fm API into a listen. """
+ track_name = scrobble.get("name")
+ track_mbid = scrobble.get("mbid")
+
+ artist = scrobble.get("artist", {})
+ artist_name = artist.get("#text")
+ artist_mbid = artist.get("mbid")
+
+ album = scrobble.get("album")
+ album_name = album.get("#text")
+ album_mbid = album.get("mbid")
+
+ if "date" in scrobble:
+ listened_at = int(scrobble["date"]["uts"])
+ listen_type = LISTEN_TYPE_IMPORT
+ listen = {"listened_at": listened_at}
+ else:
+ # todo: check now playing @attr
+ listen_type = LISTEN_TYPE_PLAYING_NOW
+ listen = {}
+
+ if not track_name or not artist_name:
+ return None, None
+
+ track_metadata = {
+ "artist_name": artist_name,
+ "track_name": track_name,
+ }
+ if album_name:
+ track_metadata["release_name"] = album_name
+
+ additional_info = {
+ "submission_client": "ListenBrainz lastfm importer v2"
+ }
+ if track_mbid:
+ additional_info["lastfm_track_mbid"] = track_mbid
+ if artist_mbid:
+ additional_info["lastfm_artist_mbid"] = artist_mbid
+ if album_mbid:
+ additional_info["lastfm_release_mbid"] = album_mbid
+
+ if additional_info:
+ track_metadata["additional_info"] = additional_info
+
+ listen["track_metadata"] = track_metadata
+ return listen, listen_type
+
+
+ def get_user_recent_tracks(self, session, user, page):
+ """ Get user’s recently played tracks from last.fm api. """
+ latest_listened_at = user["latest_listened_at"] or LISTEN_MINIMUM_DATE
+ params = {
+ "method": "user.getrecenttracks",
+ "format": "json",
+ "api_key": current_app.config["LASTFM_API_KEY"],
+ "limit": 200,
+ "user": user["external_user_id"],
+ "from": int(latest_listened_at.timestamp()),
+ "page": page
+ }
+ response = session.get(current_app.config["LASTFM_API_URL"], params=params)
+ match response.status_code:
+ case 200:
+ return response.json()
+ case 404:
+ raise ExternalServiceError("Last.FM user with username %s not found" % (params["user"],))
+ case 429:
+ raise ExternalServiceError("Encountered a rate limit.")
+ case _:
+ raise ExternalServiceAPIError('Error from the lastfm API while getting listens: %s' % (str(response.text),))
+
+
+ def process_one_user(self, user: dict) -> int:
+ """ Get recently played songs for this user and submit them to ListenBrainz.
+
+ Returns:
+ the number of recently played listens imported for the user
+ """
+ try:
+ imported_listen_count = 0
+ session = requests.Session()
+ session.mount(
+ "https://",
+ HTTPAdapter(max_retries=Retry(
+ total=3,
+ backoff_factor=1,
+ allowed_methods=["GET"],
+ # retry on 400 because last.fm wraps some service errors in 400 errors
+ status_forcelist=[400, 413, 429, 500, 502, 503, 504]
+ ))
+ )
+
+ response = self.get_user_recent_tracks(session, user, page=1)
+ pages = int(response["recenttracks"]["@attr"]["totalPages"])
+
+ for page in range(pages, 0, -1):
+ current_app.logger.info("Processing page %s", page)
+ response = self.get_user_recent_tracks(session, user, page)
+ now_playing_listen, listens, latest_listened_at = self.parse_and_validate_listen_items(
+ self.convert_scrobble_to_listen,
+ response["recenttracks"]["track"]
+ )
+
+ if now_playing_listen is not None:
+ self.submit_listens_to_listenbrainz(user, [now_playing_listen], listen_type=LISTEN_TYPE_PLAYING_NOW)
+ current_app.logger.info('imported now playing listen for %s' % (str(user['musicbrainz_id']),))
+ imported_listen_count += 1
+
+ if listens:
+ self.submit_listens_to_listenbrainz(user, listens, listen_type=LISTEN_TYPE_IMPORT)
+ self.service.update_latest_listen_ts(user['user_id'], latest_listened_at)
+ current_app.logger.info('imported %d listens for %s' % (len(listens), str(user['musicbrainz_id'])))
+ imported_listen_count += len(listens)
+
+ return imported_listen_count
+ except ExternalServiceAPIError as e:
+ # if it is an error from the Spotify API, show the error message to the user
+ self.service.update_user_import_status(user_id=user['user_id'], error=str(e))
+ if not current_app.config['TESTING']:
+ self.notify_error(user['musicbrainz_id'], str(e))
+ raise e
+
+ def process_all_users(self):
+ # todo: last.fm is prone to errors, especially for entire history imports. currently doing alternate passes
+ # where we ignore and reattempt
+ result = super().process_all_users()
+ self.exclude_error = not self.exclude_error
+ return result
+
+
+if __name__ == '__main__':
+ app = create_app()
+ with app.app_context():
+ importer = LastfmImporter()
+ importer.main()
diff --git a/listenbrainz/listens_importer/spotify.py b/listenbrainz/listens_importer/spotify.py
new file mode 100644
index 0000000000..ba508715d6
--- /dev/null
+++ b/listenbrainz/listens_importer/spotify.py
@@ -0,0 +1,294 @@
+#!/usr/bin/python3
+
+import time
+
+import spotipy
+from dateutil import parser
+from flask import current_app
+from spotipy import SpotifyException
+
+from listenbrainz.domain.external_service import ExternalServiceError, ExternalServiceAPIError, \
+ ExternalServiceInvalidGrantError
+from listenbrainz.domain.spotify import SpotifyService
+
+from listenbrainz.listens_importer.base import ListensImporter
+from listenbrainz.webserver import create_app
+from listenbrainz.webserver.views.api_tools import LISTEN_TYPE_IMPORT, LISTEN_TYPE_PLAYING_NOW
+
+
+class SpotifyImporter(ListensImporter):
+
+ def __init__(self):
+ super(SpotifyImporter, self).__init__(
+ name='spotify_reader',
+ user_friendly_name="Spotify",
+ service=SpotifyService(),
+ )
+
+ @staticmethod
+ def _convert_spotify_play_to_listen(play, listen_type):
+ """ Converts data retrieved from the Spotify API into a listen.
+
+ Args:
+ play (dict): a dict that represents a listen retrieved from Spotify
+ , this should be an "item" from the spotify response.
+ listen_type: the type of the listen (import or playing_now)
+
+ Returns:
+ listen (dict): dict that can be submitted to ListenBrainz
+ """
+ if listen_type == LISTEN_TYPE_PLAYING_NOW:
+ track = play
+ listen = {}
+ else:
+ track = play['track']
+ listen = {
+ 'listened_at': parser.parse(play['played_at']).timestamp(),
+ }
+
+ if track is None:
+ return None
+
+ artists = track.get('artists', [])
+ artist_names = []
+ spotify_artist_ids = []
+ for a in artists:
+ name = a.get('name')
+ if name is not None:
+ artist_names.append(name)
+ spotify_id = a.get('external_urls', {}).get('spotify')
+ if spotify_id is not None:
+ spotify_artist_ids.append(spotify_id)
+ artist_name = ', '.join(artist_names)
+
+ album = track.get('album', {})
+ album_artists = album.get('artists', [])
+ release_artist_names = []
+ spotify_album_artist_ids = []
+ for a in album_artists:
+ name = a.get('name')
+ if name is not None:
+ release_artist_names.append(name)
+ spotify_id = a.get('external_urls', {}).get('spotify')
+ if spotify_id is not None:
+ spotify_album_artist_ids.append(spotify_id)
+ album_artist_name = ', '.join(release_artist_names)
+
+ additional = {
+ 'tracknumber': track.get('track_number'),
+ 'spotify_artist_ids': spotify_artist_ids,
+ 'artist_names': artist_names,
+ 'discnumber': track.get('disc_number'),
+ 'duration_ms': track.get('duration_ms'),
+ 'spotify_album_id': album.get('external_urls', {}).get('spotify'),
+ # Named 'release_*' because 'release_name' is an official name in the docs
+ 'release_artist_name': album_artist_name,
+ 'release_artist_names': release_artist_names,
+ # Named 'album_*' because Spotify calls it album and this is spotify-specific
+ 'spotify_album_artist_ids': spotify_album_artist_ids,
+ 'submission_client': 'listenbrainz',
+ 'music_service': 'spotify.com'
+ }
+ isrc = track.get('external_ids', {}).get('isrc')
+ spotify_url = track.get('external_urls', {}).get('spotify')
+ if isrc:
+ additional['isrc'] = isrc
+ if spotify_url:
+ additional['spotify_id'] = spotify_url
+ additional['origin_url'] = spotify_url
+
+ listen['track_metadata'] = {
+ 'artist_name': artist_name,
+ 'track_name': track['name'],
+ 'release_name': album['name'],
+ 'additional_info': additional,
+ }
+ return listen, listen_type
+
+ def convert_spotify_current_play_to_listen(self, play):
+ return self._convert_spotify_play_to_listen(play, LISTEN_TYPE_PLAYING_NOW)
+
+ def convert_spotify_recent_play_to_listen(self, play):
+ return self._convert_spotify_play_to_listen(play, LISTEN_TYPE_IMPORT)
+
+ def make_api_request(self, user: dict, endpoint: str, **kwargs):
+ """ Make an request to the Spotify API for particular user at specified endpoint with args.
+
+ Args:
+ user: the user whose plays are to be imported.
+ endpoint: the name of Spotipy function which makes request to the required API endpoint
+
+ Returns:
+ the response from the spotify API
+
+ Raises:
+ ExternalServiceAPIError: if we encounter errors from the Spotify API.
+ ExternalServiceError: if we encounter a rate limit, even after retrying.
+ """
+ retries = 10
+ delay = 1
+ tried_to_refresh_token = False
+
+ while retries > 0:
+ try:
+ spotipy_client = spotipy.Spotify(auth=user['access_token'])
+ spotipy_call = getattr(spotipy_client, endpoint)
+ recently_played = spotipy_call(**kwargs)
+ return recently_played
+ except (AttributeError, TypeError):
+ current_app.logger.critical("Invalid spotipy endpoint or arguments:", exc_info=True)
+ return None
+ except SpotifyException as e:
+ retries -= 1
+ if e.http_status == 429:
+ # Rate Limit Problems -- the client handles these, but it can still give up
+ # after a certain number of retries, so we look at the header and try the
+ # request again, if the error is raised
+ try:
+ time_to_sleep = int(e.headers.get('Retry-After', delay))
+ except ValueError:
+ time_to_sleep = delay
+ current_app.logger.warning('Encountered a rate limit, sleeping %d seconds and trying again...',
+ time_to_sleep)
+ time.sleep(time_to_sleep)
+ delay += 1
+ if retries == 0:
+ raise ExternalServiceError('Encountered a rate limit.')
+
+ elif e.http_status in (400, 403):
+ current_app.logger.critical('Error from the Spotify API for user %s: %s', user['musicbrainz_id'],
+ str(e), exc_info=True)
+ raise ExternalServiceAPIError('Error from the Spotify API while getting listens: %s', str(e))
+ elif e.http_status >= 500 and e.http_status < 600:
+ # these errors are not our fault, most probably. so just log them and retry.
+ current_app.logger.error('Error while trying to get listens for user %s: %s',
+ user['musicbrainz_id'], str(e), exc_info=True)
+ if retries == 0:
+ raise ExternalServiceAPIError('Error from the spotify API while getting listens: %s', str(e))
+
+ elif e.http_status == 401:
+ # if we get 401 Unauthorized from Spotify, that means our token might have expired.
+ # In that case, try to refresh the token, if there is an error even while refreshing
+ # give up and report to the user.
+ # We only try to refresh the token once, if we still get 401 after that, we give up.
+ if not tried_to_refresh_token:
+ user = SpotifyService().refresh_access_token(user['user_id'], user['refresh_token'])
+ tried_to_refresh_token = True
+
+ else:
+ raise ExternalServiceAPIError(
+ 'Could not authenticate with Spotify, please unlink and link your account again.')
+ elif e.http_status == 404:
+ current_app.logger.error("404 while trying to get listens for user %s", str(user), exc_info=True)
+ if retries == 0:
+ raise ExternalServiceError("404 while trying to get listens for user %s" % str(user))
+ except Exception as e:
+ retries -= 1
+ current_app.logger.error('Unexpected error while getting listens: %s', str(e), exc_info=True)
+ if retries == 0:
+ raise ExternalServiceError('Unexpected error while getting listens: %s' % str(e))
+
+ def get_user_recently_played(self, user):
+ """ Get tracks from the current user’s recently played tracks. """
+ latest_listened_at_ts = 0
+ if user['latest_listened_at']:
+ latest_listened_at_ts = int(user['latest_listened_at'].timestamp() * 1000) # latest listen UNIX ts in ms
+
+ return self.make_api_request(user, 'current_user_recently_played', limit=50, after=latest_listened_at_ts)
+
+ def get_user_currently_playing(self, user):
+ """ Get the user's currently playing track.
+ """
+ return self.make_api_request(user, 'current_user_playing_track')
+
+ def process_one_user(self, user: dict) -> int:
+ """ Get recently played songs for this user and submit them to ListenBrainz.
+
+ Args:
+ user (spotify.Spotify): the user whose plays are to be imported.
+
+ Raises:
+ spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
+ spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
+ or if we get errors while submitting the data to ListenBrainz
+ Returns:
+ the number of recently played listens imported for the user
+ """
+ try:
+ if self.service.user_oauth_token_has_expired(user):
+ user = self.service.refresh_access_token(user['user_id'], user['refresh_token'])
+
+ listens = []
+ latest_listened_at = None
+
+ # If there is no playback, currently_playing will be None.
+ # There are two playing types, track and episode. We use only the
+ # track type. Therefore, when the user's playback type is not a track,
+ # Spotify will set the item field to null which becomes None after
+ # parsing the JSON. Due to these reasons, we cannot simplify the
+ # checks below.
+ currently_playing = self.get_user_currently_playing(user)
+ if currently_playing is not None:
+ currently_playing_item = currently_playing.get('item', None)
+ if currently_playing_item is not None:
+ current_app.logger.debug('Received a currently playing track for %s', str(user))
+ now_playing_listen, _, _ = self.parse_and_validate_listen_items(
+ self.convert_spotify_current_play_to_listen,
+ [currently_playing_item]
+ )
+ if now_playing_listen:
+ self.submit_listens_to_listenbrainz(user, [now_playing_listen], listen_type=LISTEN_TYPE_PLAYING_NOW)
+
+ recently_played = self.get_user_recently_played(user)
+ if recently_played is not None and 'items' in recently_played:
+ _, listens, latest_listened_at = self.parse_and_validate_listen_items(
+ self.convert_spotify_recent_play_to_listen,
+ recently_played['items']
+ )
+ current_app.logger.debug('Received %d tracks for %s', len(listens), str(user))
+
+ # if we don't have any new listens, return. we don't check whether the listens list is empty here
+ # because it will empty in both cases where we don't receive any listens and when we receive only
+ # bad listens. so instead we check latest_listened_at which is None only in case when we received
+ # nothing from spotify.
+ if latest_listened_at is None:
+ self.service.update_user_import_status(user['user_id'])
+ return 0
+
+ self.submit_listens_to_listenbrainz(user, listens, listen_type=LISTEN_TYPE_IMPORT)
+
+ # we've succeeded so update the last_updated and latest_listened_at field for this user
+ self.service.update_latest_listen_ts(user['user_id'], latest_listened_at)
+
+ current_app.logger.info('imported %d listens for %s' % (len(listens), str(user['musicbrainz_id'])))
+ return len(listens)
+
+ except ExternalServiceInvalidGrantError:
+ error_message = "It seems like you've revoked permission for us to read your spotify account"
+ self.service.update_user_import_status(user_id=user['user_id'], error=error_message)
+ if not current_app.config['TESTING']:
+ self.notify_error(user['musicbrainz_id'], error_message)
+ # user has revoked authorization through spotify ui or deleted their spotify account etc.
+ #
+ # we used to remove spotify access tokens from our database whenever we detected token revocation
+ # at one point. but one day spotify had a downtime while resulted in false revocation errors, and
+ # we ended up deleting most of our users' spotify access tokens. now we don't remove the token from
+ # database. this is actually more resilient and without downsides. if a user actually revoked their
+ # token, then its useless anyway so doesn't matter if we remove it. and if it is a false revocation
+ # error, we are saved! :) in any case, we do set an error message for the user in the database
+ # so that we can skip in future runs and notify them to reconnect if they want.
+ raise ExternalServiceError("User has revoked spotify authorization")
+
+ except ExternalServiceAPIError as e:
+ # if it is an error from the Spotify API, show the error message to the user
+ self.service.update_user_import_status(user_id=user['user_id'], error=str(e))
+ if not current_app.config['TESTING']:
+ self.notify_error(user['musicbrainz_id'], str(e))
+ raise ExternalServiceError("Could not refresh user token from spotify")
+
+
+if __name__ == '__main__':
+ app = create_app()
+ with app.app_context():
+ importer = SpotifyImporter()
+ importer.main()
diff --git a/listenbrainz/spotify_updater/tests/__init__.py b/listenbrainz/listens_importer/tests/__init__.py
similarity index 100%
rename from listenbrainz/spotify_updater/tests/__init__.py
rename to listenbrainz/listens_importer/tests/__init__.py
diff --git a/listenbrainz/spotify_updater/tests/data/spotify_play_no_isrc.json b/listenbrainz/listens_importer/tests/data/spotify_play_no_isrc.json
similarity index 100%
rename from listenbrainz/spotify_updater/tests/data/spotify_play_no_isrc.json
rename to listenbrainz/listens_importer/tests/data/spotify_play_no_isrc.json
diff --git a/listenbrainz/spotify_updater/tests/data/spotify_play_two_artists.json b/listenbrainz/listens_importer/tests/data/spotify_play_two_artists.json
similarity index 100%
rename from listenbrainz/spotify_updater/tests/data/spotify_play_two_artists.json
rename to listenbrainz/listens_importer/tests/data/spotify_play_two_artists.json
diff --git a/listenbrainz/spotify_updater/tests/test_spotify_read_listens.py b/listenbrainz/listens_importer/tests/test_spotify_read_listens.py
similarity index 85%
rename from listenbrainz/spotify_updater/tests/test_spotify_read_listens.py
rename to listenbrainz/listens_importer/tests/test_spotify_read_listens.py
index 3287d6f7bb..0402633e4e 100644
--- a/listenbrainz/spotify_updater/tests/test_spotify_read_listens.py
+++ b/listenbrainz/listens_importer/tests/test_spotify_read_listens.py
@@ -10,8 +10,7 @@
from listenbrainz.domain.external_service import ExternalServiceAPIError, \
ExternalServiceInvalidGrantError
from listenbrainz.domain.spotify import SpotifyService
-from listenbrainz.spotify_updater import spotify_read_listens
-from listenbrainz.webserver.views.api_tools import LISTEN_TYPE_IMPORT
+from listenbrainz.listens_importer.spotify import SpotifyImporter
from unittest.mock import patch
from listenbrainz.db.testing import DatabaseTestCase
from listenbrainz.db import external_service_oauth as db_oauth
@@ -32,7 +31,9 @@ def setUp(self):
def test_parse_play_to_listen_no_isrc(self):
data = json.load(open(os.path.join(self.DATA_DIR, 'spotify_play_no_isrc.json')))
- listen = spotify_read_listens._convert_spotify_play_to_listen(data, LISTEN_TYPE_IMPORT)
+ with listenbrainz.webserver.create_app().app_context():
+ importer = SpotifyImporter()
+ listen = importer.convert_spotify_recent_play_to_listen(data)[0]
expected_listen = {
'listened_at': 1519241031.761,
@@ -66,7 +67,9 @@ def test_parse_play_to_listen_many_artists(self):
# If a spotify play record has many artists, make sure they are appended
data = json.load(open(os.path.join(self.DATA_DIR, 'spotify_play_two_artists.json')))
- listen = spotify_read_listens._convert_spotify_play_to_listen(data, LISTEN_TYPE_IMPORT)
+ with listenbrainz.webserver.create_app().app_context():
+ importer = SpotifyImporter()
+ listen = importer.convert_spotify_recent_play_to_listen(data)[0]
expected_listen = {
'listened_at': 1519240503.665,
@@ -97,25 +100,27 @@ def test_parse_play_to_listen_many_artists(self):
self.assertDictEqual(listen, expected_listen)
- @patch('listenbrainz.spotify_updater.spotify_read_listens.send_mail')
+ @patch('listenbrainz.listens_importer.base.send_mail')
def test_notify_user(self, mock_send_mail):
db_user.create(self.db_conn, 2, "two", "one@two.one")
app = listenbrainz.webserver.create_app()
app.config['SERVER_NAME'] = "test"
with app.app_context():
- spotify_read_listens.notify_error(musicbrainz_id="two", error='some random error')
+ importer = SpotifyImporter()
+ importer.notify_error(musicbrainz_id="two", error='some random error')
mock_send_mail.assert_called_once()
self.assertListEqual(mock_send_mail.call_args[1]['recipients'], ['one@two.one'])
@patch('listenbrainz.domain.spotify.SpotifyService.update_user_import_status')
- @patch('listenbrainz.spotify_updater.spotify_read_listens.notify_error')
- @patch('listenbrainz.spotify_updater.spotify_read_listens.make_api_request')
+ @patch.object(SpotifyImporter, 'notify_error')
+ @patch.object(SpotifyImporter, 'make_api_request')
def test_notification_on_api_error(self, mock_make_api_request, mock_notify_error, mock_update):
mock_make_api_request.side_effect = ExternalServiceAPIError('api borked')
app = listenbrainz.webserver.create_app()
app.config['TESTING'] = False
with app.app_context():
- spotify_read_listens.process_all_spotify_users()
+ importer = SpotifyImporter()
+ importer.process_all_users()
mock_notify_error.assert_called_once_with(self.user['musicbrainz_id'], 'api borked')
mock_update.assert_called_once()
@@ -126,7 +131,8 @@ def test_spotipy_methods_are_called_with_correct_params(self, mock_spotipy):
with listenbrainz.webserver.create_app().app_context():
SpotifyService().update_latest_listen_ts(self.user['id'],
int(datetime(2014, 5, 13, 16, 53, 20).timestamp()))
- spotify_read_listens.process_all_spotify_users()
+ importer = SpotifyImporter()
+ importer.process_all_users()
mock_spotipy.return_value.current_user_playing_track.assert_called_once()
mock_spotipy.return_value.current_user_recently_played.assert_called_once_with(limit=50, after=1400000000000)
@@ -137,7 +143,8 @@ def test_spotipy_methods_are_called_with_correct_params_with_no_latest_listened_
mock_current_user_recently_played.return_value = None
with listenbrainz.webserver.create_app().app_context():
- spotify_read_listens.process_all_spotify_users()
+ importer = SpotifyImporter()
+ importer.process_all_users()
mock_current_user_playing_track.assert_called_once()
mock_current_user_recently_played.assert_called_once_with(limit=50, after=0)
@@ -155,5 +162,7 @@ def process_one_user(self, mock_refresh_user_token):
latest_listened_at=None,
scopes=['user-read-recently-played'],
)
- with self.assertRaises(ExternalServiceInvalidGrantError):
- spotify_read_listens.process_one_user(expired_token_spotify_user, SpotifyService())
+ with (self.assertRaises(ExternalServiceInvalidGrantError),
+ listenbrainz.webserver.create_app().app_context()):
+ importer = SpotifyImporter()
+ importer.process_one_user(expired_token_spotify_user)
diff --git a/listenbrainz/spotify_updater/spotify_read_listens.py b/listenbrainz/spotify_updater/spotify_read_listens.py
deleted file mode 100644
index 6d5485f189..0000000000
--- a/listenbrainz/spotify_updater/spotify_read_listens.py
+++ /dev/null
@@ -1,430 +0,0 @@
-#!/usr/bin/python3
-import time
-from typing import Dict, List
-
-import spotipy
-from brainzutils import metrics
-from brainzutils.mail import send_mail
-from dateutil import parser
-from flask import current_app, render_template
-from spotipy import SpotifyException
-from sqlalchemy.exc import DatabaseError, SQLAlchemyError
-from werkzeug.exceptions import InternalServerError, ServiceUnavailable
-
-import listenbrainz.webserver
-from listenbrainz.db import user as db_user
-from listenbrainz.db.exceptions import DatabaseException
-from listenbrainz.domain.external_service import ExternalServiceError, ExternalServiceAPIError, \
- ExternalServiceInvalidGrantError
-from listenbrainz.domain.spotify import SpotifyService
-from listenbrainz.webserver.errors import ListenValidationError
-from listenbrainz.webserver.models import SubmitListenUserMetadata
-from listenbrainz.webserver.views.api_tools import insert_payload, validate_listen, LISTEN_TYPE_IMPORT, \
- LISTEN_TYPE_PLAYING_NOW
-
-METRIC_UPDATE_INTERVAL = 60 # seconds
-_listens_imported_since_last_update = 0 # number of listens imported since last metric update was submitted
-_metric_submission_time = time.monotonic() + METRIC_UPDATE_INTERVAL
-
-
-def notify_error(musicbrainz_id: str, error: str):
- """ Notifies specified user via email about error during Spotify import.
-
- Args:
- musicbrainz_id: the MusicBrainz ID of the user
- error: a description of the error encountered.
- """
- user_email = db_user.get_by_mb_id(listenbrainz.webserver.db_conn, musicbrainz_id, fetch_email=True)["email"]
- if not user_email:
- return
-
- spotify_url = current_app.config['SERVER_ROOT_URL'] + '/settings/music-services/details/'
- text = render_template('emails/spotify_import_error.txt', error=error, link=spotify_url)
- send_mail(
- subject='ListenBrainz Spotify Importer Error',
- text=text,
- recipients=[user_email],
- from_name='ListenBrainz',
- from_addr='noreply@'+current_app.config['MAIL_FROM_DOMAIN'],
- )
-
-
-def _convert_spotify_play_to_listen(play, listen_type):
- """ Converts data retrieved from the Spotify API into a listen.
-
- Args:
- play (dict): a dict that represents a listen retrieved from Spotify
- , this should be an "item" from the spotify response.
- listen_type: the type of the listen (import or playing_now)
-
- Returns:
- listen (dict): dict that can be submitted to ListenBrainz
- """
- if listen_type == LISTEN_TYPE_PLAYING_NOW:
- track = play
- listen = {}
- else:
- track = play['track']
- listen = {
- 'listened_at': parser.parse(play['played_at']).timestamp(),
- }
-
- if track is None:
- return None
-
- artists = track.get('artists', [])
- artist_names = []
- spotify_artist_ids = []
- for a in artists:
- name = a.get('name')
- if name is not None:
- artist_names.append(name)
- spotify_id = a.get('external_urls', {}).get('spotify')
- if spotify_id is not None:
- spotify_artist_ids.append(spotify_id)
- artist_name = ', '.join(artist_names)
-
- album = track.get('album', {})
- album_artists = album.get('artists', [])
- release_artist_names = []
- spotify_album_artist_ids = []
- for a in album_artists:
- name = a.get('name')
- if name is not None:
- release_artist_names.append(name)
- spotify_id = a.get('external_urls', {}).get('spotify')
- if spotify_id is not None:
- spotify_album_artist_ids.append(spotify_id)
- album_artist_name = ', '.join(release_artist_names)
-
- additional = {
- 'tracknumber': track.get('track_number'),
- 'spotify_artist_ids': spotify_artist_ids,
- 'artist_names': artist_names,
- 'discnumber': track.get('disc_number'),
- 'duration_ms': track.get('duration_ms'),
- 'spotify_album_id': album.get('external_urls', {}).get('spotify'),
- # Named 'release_*' because 'release_name' is an official name in the docs
- 'release_artist_name': album_artist_name,
- 'release_artist_names': release_artist_names,
- # Named 'album_*' because Spotify calls it album and this is spotify-specific
- 'spotify_album_artist_ids': spotify_album_artist_ids,
- 'submission_client': 'listenbrainz',
- 'music_service': 'spotify.com'
- }
- isrc = track.get('external_ids', {}).get('isrc')
- spotify_url = track.get('external_urls', {}).get('spotify')
- if isrc:
- additional['isrc'] = isrc
- if spotify_url:
- additional['spotify_id'] = spotify_url
- additional['origin_url'] = spotify_url
-
- listen['track_metadata'] = {
- 'artist_name': artist_name,
- 'track_name': track['name'],
- 'release_name': album['name'],
- 'additional_info': additional,
- }
- return listen
-
-
-def make_api_request(user: dict, endpoint: str, **kwargs):
- """ Make an request to the Spotify API for particular user at specified endpoint with args.
-
- Args:
- user: the user whose plays are to be imported.
- endpoint: the name of Spotipy function which makes request to the required API endpoint
-
- Returns:
- the response from the spotify API
-
- Raises:
- ExternalServiceAPIError: if we encounter errors from the Spotify API.
- ExternalServiceError: if we encounter a rate limit, even after retrying.
- """
- retries = 10
- delay = 1
- tried_to_refresh_token = False
-
- while retries > 0:
- try:
- spotipy_client = spotipy.Spotify(auth=user['access_token'])
- spotipy_call = getattr(spotipy_client, endpoint)
- recently_played = spotipy_call(**kwargs)
- break
- except (AttributeError, TypeError):
- current_app.logger.critical("Invalid spotipy endpoint or arguments:", exc_info=True)
- return None
- except SpotifyException as e:
- retries -= 1
- if e.http_status == 429:
- # Rate Limit Problems -- the client handles these, but it can still give up
- # after a certain number of retries, so we look at the header and try the
- # request again, if the error is raised
- try:
- time_to_sleep = int(e.headers.get('Retry-After', delay))
- except ValueError:
- time_to_sleep = delay
- current_app.logger.warn('Encountered a rate limit, sleeping %d seconds and trying again...', time_to_sleep)
- time.sleep(time_to_sleep)
- delay += 1
- if retries == 0:
- raise ExternalServiceError('Encountered a rate limit.')
-
- elif e.http_status in (400, 403):
- current_app.logger.critical('Error from the Spotify API for user %s: %s', user['musicbrainz_id'], str(e), exc_info=True)
- raise ExternalServiceAPIError('Error from the Spotify API while getting listens: %s', str(e))
-
- elif e.http_status >= 500 and e.http_status < 600:
- # these errors are not our fault, most probably. so just log them and retry.
- current_app.logger.error('Error while trying to get listens for user %s: %s', user['musicbrainz_id'], str(e), exc_info=True)
- if retries == 0:
- raise ExternalServiceAPIError('Error from the spotify API while getting listens: %s', str(e))
-
- elif e.http_status == 401:
- # if we get 401 Unauthorized from Spotify, that means our token might have expired.
- # In that case, try to refresh the token, if there is an error even while refreshing
- # give up and report to the user.
- # We only try to refresh the token once, if we still get 401 after that, we give up.
- if not tried_to_refresh_token:
- user = SpotifyService().refresh_access_token(user['user_id'], user['refresh_token'])
- tried_to_refresh_token = True
-
- else:
- raise ExternalServiceAPIError('Could not authenticate with Spotify, please unlink and link your account again.')
- elif e.http_status == 404:
- current_app.logger.error("404 while trying to get listens for user %s", str(user), exc_info=True)
- if retries == 0:
- raise ExternalServiceError("404 while trying to get listens for user %s" % str(user))
- except Exception as e:
- retries -= 1
- current_app.logger.error('Unexpected error while getting listens: %s', str(e), exc_info=True)
- if retries == 0:
- raise ExternalServiceError('Unexpected error while getting listens: %s' % str(e))
-
- return recently_played
-
-
-def get_user_recently_played(user):
- """ Get tracks from the current user’s recently played tracks.
- """
- latest_listened_at_ts = 0
- if user['latest_listened_at']:
- latest_listened_at_ts = int(user['latest_listened_at'].timestamp() * 1000) # latest listen UNIX ts in ms
-
- return make_api_request(user, 'current_user_recently_played', limit=50, after=latest_listened_at_ts)
-
-
-def get_user_currently_playing(user):
- """ Get the user's currently playing track.
- """
- return make_api_request(user, 'current_user_playing_track')
-
-
-def submit_listens_to_listenbrainz(user: Dict, listens: List, listen_type=LISTEN_TYPE_IMPORT):
- """ Submit a batch of listens to ListenBrainz
-
- Args:
- user: the user whose listens are to be submitted, dict should contain
- at least musicbrainz_id and user_id
- listens: a list of listens to be submitted
- listen_type: the type of listen (single, import, playing_now)
- """
- username = user['musicbrainz_id']
- user_metadata = SubmitListenUserMetadata(user_id=user['user_id'], musicbrainz_id=username)
- retries = 10
- while retries >= 0:
- try:
- current_app.logger.debug('Submitting %d listens for user %s', len(listens), username)
- insert_payload(listens, user_metadata, listen_type=listen_type)
- current_app.logger.debug('Submitted!')
- break
- except (InternalServerError, ServiceUnavailable) as e:
- retries -= 1
- current_app.logger.error('ISE while trying to import listens for %s: %s', username, str(e))
- if retries == 0:
- raise ExternalServiceError('ISE while trying to import listens: %s', str(e))
-
-
-def parse_and_validate_spotify_plays(plays, listen_type):
- """ Converts and validates the listens received from the Spotify API.
-
- Args:
- plays: a list of items received from Spotify
- listen_type: the type of the plays (import or playing now)
-
- Returns:
- tuple of (a list of valid listens to submit to ListenBrainz, timestamp of latest listen)
- """
- listens = []
- latest_listen_ts = None
-
- for play in plays:
- listen = _convert_spotify_play_to_listen(play, listen_type=listen_type)
-
- if listen_type == LISTEN_TYPE_IMPORT and \
- (latest_listen_ts is None or listen['listened_at'] > latest_listen_ts):
- latest_listen_ts = listen['listened_at']
-
- try:
- listens.append(validate_listen(listen, listen_type))
- except ListenValidationError:
- pass
- return listens, latest_listen_ts
-
-
-def process_one_user(user: dict, service: SpotifyService) -> int:
- """ Get recently played songs for this user and submit them to ListenBrainz.
-
- Args:
- user (spotify.Spotify): the user whose plays are to be imported.
- service (listenbrainz.domain.spotify.SpotifyService): service to process users
-
- Raises:
- spotify.SpotifyAPIError: if we encounter errors from the Spotify API.
- spotify.SpotifyListenBrainzError: if we encounter a rate limit, even after retrying.
- or if we get errors while submitting the data to ListenBrainz
- Returns:
- the number of recently played listens imported for the user
- """
- try:
- if service.user_oauth_token_has_expired(user):
- user = service.refresh_access_token(user['user_id'], user['refresh_token'])
-
- listens = []
- latest_listened_at = None
-
- # If there is no playback, currently_playing will be None.
- # There are two playing types, track and episode. We use only the
- # track type. Therefore, when the user's playback type is not a track,
- # Spotify will set the item field to null which becomes None after
- # parsing the JSON. Due to these reasons, we cannot simplify the
- # checks below.
- currently_playing = get_user_currently_playing(user)
- if currently_playing is not None:
- currently_playing_item = currently_playing.get('item', None)
- if currently_playing_item is not None:
- current_app.logger.debug('Received a currently playing track for %s', str(user))
- listens, latest_listened_at = parse_and_validate_spotify_plays(
- [currently_playing_item],
- LISTEN_TYPE_PLAYING_NOW
- )
- if listens:
- submit_listens_to_listenbrainz(user, listens, listen_type=LISTEN_TYPE_PLAYING_NOW)
-
- recently_played = get_user_recently_played(user)
- if recently_played is not None and 'items' in recently_played:
- listens, latest_listened_at = parse_and_validate_spotify_plays(recently_played['items'], LISTEN_TYPE_IMPORT)
- current_app.logger.debug('Received %d tracks for %s', len(listens), str(user))
-
- # if we don't have any new listens, return. we don't check whether the listens list is empty here
- # because it will empty in both cases where we don't receive any listens and when we receive only
- # bad listens. so instead we check latest_listened_at which is None only in case when we received
- # nothing from spotify.
- if latest_listened_at is None:
- service.update_user_import_status(user['user_id'])
- return 0
-
- submit_listens_to_listenbrainz(user, listens, listen_type=LISTEN_TYPE_IMPORT)
-
- # we've succeeded so update the last_updated and latest_listened_at field for this user
- service.update_latest_listen_ts(user['user_id'], latest_listened_at)
-
- current_app.logger.info('imported %d listens for %s' % (len(listens), str(user['musicbrainz_id'])))
- return len(listens)
-
- except ExternalServiceInvalidGrantError:
- error_message = "It seems like you've revoked permission for us to read your spotify account"
- service.update_user_import_status(user_id=user['user_id'], error=error_message)
- if not current_app.config['TESTING']:
- notify_error(user['musicbrainz_id'], error_message)
- # user has revoked authorization through spotify ui or deleted their spotify account etc.
- #
- # we used to remove spotify access tokens from our database whenever we detected token revocation
- # at one point. but one day spotify had a downtime while resulted in false revocation errors, and
- # we ended up deleting most of our users' spotify access tokens. now we don't remove the token from
- # database. this is actually more resilient and without downsides. if a user actually revoked their
- # token, then its useless anyway so doesn't matter if we remove it. and if it is a false revocation
- # error, we are saved! :) in any case, we do set an error message for the user in the database
- # so that we can skip in future runs and notify them to reconnect if they want.
- raise ExternalServiceError("User has revoked spotify authorization")
-
- except ExternalServiceAPIError as e:
- # if it is an error from the Spotify API, show the error message to the user
- service.update_user_import_status(user_id=user['user_id'], error=str(e))
- if not current_app.config['TESTING']:
- notify_error(user['musicbrainz_id'], str(e))
- raise ExternalServiceError("Could not refresh user token from spotify")
-
-
-def process_all_spotify_users():
- """ Get a batch of users to be processed and import their Spotify plays.
-
- Returns:
- (success, failure) where
- success: the number of users whose plays were successfully imported.
- failure: the number of users for whom we faced errors while importing.
- """
-
- global _listens_imported_since_last_update, _metric_submission_time
-
- service = SpotifyService()
- try:
- users = service.get_active_users_to_process()
- except DatabaseException as e:
- listenbrainz.webserver.db_conn.rollback()
- current_app.logger.error('Cannot get list of users due to error %s', str(e), exc_info=True)
- return 0, 0
-
- if not users:
- return 0, 0
-
- current_app.logger.info('Process %d users...' % len(users))
- success = 0
- failure = 0
- for u in users:
- try:
- _listens_imported_since_last_update += process_one_user(u, service)
- success += 1
- except (DatabaseException, DatabaseError, SQLAlchemyError):
- listenbrainz.webserver.db_conn.rollback()
- current_app.logger.error('spotify_reader could not import listens for user %s:',
- u['musicbrainz_id'], exc_info=True)
- except Exception:
- current_app.logger.error('spotify_reader could not import listens for user %s:',
- u['musicbrainz_id'], exc_info=True)
- failure += 1
-
- if time.monotonic() > _metric_submission_time:
- _metric_submission_time += METRIC_UPDATE_INTERVAL
- metrics.set("spotify_reader", imported_listens=_listens_imported_since_last_update)
- _listens_imported_since_last_update = 0
-
- current_app.logger.info('Processed %d users successfully!', success)
- current_app.logger.info('Encountered errors while processing %d users.', failure)
- return success, failure
-
-
-def main():
- app = listenbrainz.webserver.create_app()
- with app.app_context():
- current_app.logger.info('Spotify Reader started...')
- while True:
- t = time.monotonic()
- success, failure = process_all_spotify_users()
- total_users = success + failure
- if total_users > 0:
- total_time = time.monotonic() - t
- avg_time = total_time / total_users
- metrics.set("spotify_reader",
- users_processed=total_users,
- time_to_process_all_users=total_time,
- time_to_process_one_user=avg_time)
- current_app.logger.info('All %d users in batch have been processed.', total_users)
- current_app.logger.info('Total time taken: %.2f s, average time per user: %.2f s.', total_time, avg_time)
- time.sleep(10)
-
-
-if __name__ == '__main__':
- main()
diff --git a/listenbrainz/tests/integration/test_spotify_read_listens.py b/listenbrainz/tests/integration/test_spotify_read_listens.py
index c00a0d2061..7b434bea8d 100644
--- a/listenbrainz/tests/integration/test_spotify_read_listens.py
+++ b/listenbrainz/tests/integration/test_spotify_read_listens.py
@@ -4,11 +4,10 @@
from unittest.mock import patch
from flask import url_for
-from flask.testing import FlaskClient
from data.model.external_service import ExternalServiceType
from listenbrainz.listenstore.timescale_utils import recalculate_all_user_data
-from listenbrainz.spotify_updater import spotify_read_listens
+from listenbrainz.listens_importer.spotify import SpotifyImporter
from listenbrainz.tests.integration import ListenAPIIntegrationTestCase
from listenbrainz.db import external_service_oauth
@@ -31,15 +30,17 @@ def tearDown(self):
self.ctx.pop()
super(SpotifyReaderTestCase, self).tearDown()
- @patch('listenbrainz.spotify_updater.spotify_read_listens.get_user_currently_playing')
- @patch('listenbrainz.spotify_updater.spotify_read_listens.get_user_recently_played')
+ @patch.object(SpotifyImporter, 'get_user_currently_playing')
+ @patch.object(SpotifyImporter, 'get_user_recently_played')
def test_spotify_recently_played_submitted(self, mock_recently_played, mock_currently_playing):
with open(self.path_to_data_file('spotify_recently_played_submitted.json')) as f:
mock_recently_played.return_value = json.load(f)
mock_currently_playing.return_value = None
- result = spotify_read_listens.process_all_spotify_users()
- self.assertEqual(result, (1, 0))
+ with self.app.app_context():
+ importer = SpotifyImporter()
+ result = importer.process_all_users()
+ self.assertEqual(result, (1, 0))
time.sleep(0.5)
recalculate_all_user_data()
diff --git a/listenbrainz/webserver/templates/emails/listens_importer_error.txt b/listenbrainz/webserver/templates/emails/listens_importer_error.txt
new file mode 100644
index 0000000000..c9d628f319
--- /dev/null
+++ b/listenbrainz/webserver/templates/emails/listens_importer_error.txt
@@ -0,0 +1,10 @@
+Hi!
+
+We encountered an error while importing your listens from {{service}}.
+The error was as follows: "{{error}}"
+
+Please take a look at your {{service}} import page ({{link}})
+for more information.
+
+Best,
+The ListenBrainz Team
diff --git a/listenbrainz/webserver/templates/emails/spotify_import_error.txt b/listenbrainz/webserver/templates/emails/spotify_import_error.txt
deleted file mode 100644
index 0576f4a11b..0000000000
--- a/listenbrainz/webserver/templates/emails/spotify_import_error.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-Hi!
-
-We encountered an error while importing your listens from Spotify.
-The error was as follows: "{{error}}"
-
-Please take a look at your Spotify import page ({{link}})
-for more information.
-
-Best,
-The ListenBrainz Team
diff --git a/listenbrainz/webserver/views/donors.py b/listenbrainz/webserver/views/donors.py
index 6b1a7c3dab..6818c7b7b6 100644
--- a/listenbrainz/webserver/views/donors.py
+++ b/listenbrainz/webserver/views/donors.py
@@ -39,7 +39,7 @@ def donors_post():
user_playlist_count = db_playlist.get_playlist_count(ts_conn, donor_ids) if donor_ids else {}
for donor in donors:
- donor_info = donors_info.get(donor["musicbrainz_id"])
+ donor_info = donors_info.get(donor["musicbrainz_id"].lower())
if not donor_info:
donor['listenCount'] = None
donor['playlistCount'] = None
diff --git a/listenbrainz/webserver/views/entity_pages.py b/listenbrainz/webserver/views/entity_pages.py
index 9d6edd813c..a913105670 100644
--- a/listenbrainz/webserver/views/entity_pages.py
+++ b/listenbrainz/webserver/views/entity_pages.py
@@ -20,6 +20,17 @@
release_group_bp = Blueprint("release-group", __name__)
+def get_release_group_sort_key(release_group):
+ """ Return a tuple that sorts release group by total_listen_count and then by date """
+ release_date = release_group.get("date")
+ if release_date is None:
+ release_date = datetime.min
+ else:
+ release_date = datetime.strptime(release_date, "%Y-%m-%d")
+
+ return release_group["total_listen_count"] or 0, release_date
+
+
def get_cover_art_for_artist(release_groups):
""" Get the cover art for an artist using a list of their release groups """
covers = []
@@ -159,6 +170,8 @@ def artist_entity(artist_mbid):
release_group["total_user_count"] = pop["total_user_count"]
release_groups.append(release_group)
+ release_groups.sort(key=get_release_group_sort_key, reverse=True)
+
listening_stats = get_entity_listener(db_conn, "artists", artist_mbid, "all_time")
if listening_stats is None:
listening_stats = {
diff --git a/listenbrainz/webserver/views/settings.py b/listenbrainz/webserver/views/settings.py
index 45c767eaa1..62a59069a3 100644
--- a/listenbrainz/webserver/views/settings.py
+++ b/listenbrainz/webserver/views/settings.py
@@ -1,36 +1,30 @@
import json
-import os.path
from datetime import datetime
-import orjson
-import json
-from flask import Blueprint, Response, render_template, request, url_for, \
- redirect, current_app, jsonify, stream_with_context, send_file
+from flask import Blueprint, render_template, request, url_for, \
+ redirect, current_app, jsonify
from flask_login import current_user, login_required
-from sqlalchemy import text
from werkzeug.exceptions import NotFound, BadRequest
-import listenbrainz.db.feedback as db_feedback
import listenbrainz.db.user as db_user
import listenbrainz.db.user_setting as db_usersetting
from data.model.external_service import ExternalServiceType
from listenbrainz.background.background_tasks import add_task
-from listenbrainz.db import listens_importer
-from listenbrainz.db.missing_musicbrainz_data import get_user_missing_musicbrainz_data
from listenbrainz.db.exceptions import DatabaseException
+from listenbrainz.db.missing_musicbrainz_data import get_user_missing_musicbrainz_data
from listenbrainz.domain.apple import AppleService
from listenbrainz.domain.critiquebrainz import CritiqueBrainzService, CRITIQUEBRAINZ_SCOPES
from listenbrainz.domain.external_service import ExternalService, ExternalServiceInvalidGrantError
+from listenbrainz.domain.lastfm import LastfmService
from listenbrainz.domain.musicbrainz import MusicBrainzService
from listenbrainz.domain.soundcloud import SoundCloudService
from listenbrainz.domain.spotify import SpotifyService, SPOTIFY_LISTEN_PERMISSIONS, SPOTIFY_IMPORT_PERMISSIONS
from listenbrainz.webserver import db_conn, ts_conn
-from listenbrainz.webserver import timescale_connection
from listenbrainz.webserver.decorators import web_listenstore_needed
-from listenbrainz.webserver.errors import APIServiceUnavailable, APINotFound, APIForbidden, APIInternalServerError
+from listenbrainz.webserver.errors import APIServiceUnavailable, APINotFound, APIForbidden, APIInternalServerError, \
+ APIBadRequest
from listenbrainz.webserver.login import api_login_required
-
settings_bp = Blueprint("settings", __name__)
@@ -67,16 +61,6 @@ def set_troi_prefs():
return jsonify(data)
-@settings_bp.route("/resetlatestimportts/", methods=["POST"])
-@api_login_required
-def reset_latest_import_timestamp():
- try:
- listens_importer.update_latest_listened_at(db_conn, current_user.id, ExternalServiceType.LASTFM, 0)
- return jsonify({"success": True})
- except DatabaseException:
- raise APIInternalServerError("Something went wrong! Unable to reset latest import timestamp right now.")
-
-
@settings_bp.route("/import/", methods=["POST"])
@api_login_required
def import_data():
@@ -88,15 +72,13 @@ def import_data():
else:
user_has_email = True
- # Return error if LASTFM_API_KEY is not given in config.py
- if 'LASTFM_API_KEY' not in current_app.config or current_app.config['LASTFM_API_KEY'] == "":
- return jsonify({"error": "LASTFM_API_KEY not specified."}), 404
+ # Return error if LIBREFM_API_KEY is not given in config.py
+ if 'LIBREFM_API_KEY' not in current_app.config or current_app.config['LIBREFM_API_KEY'] == "":
+ return jsonify({"error": "LIBREFM_API_KEY not specified."}), 404
data = {
"user_has_email": user_has_email,
"profile_url": url_for('user.index', path="", user_name=current_user.musicbrainz_id),
- "lastfm_api_url": current_app.config["LASTFM_API_URL"],
- "lastfm_api_key": current_app.config["LASTFM_API_KEY"],
"librefm_api_url": current_app.config["LIBREFM_API_URL"],
"librefm_api_key": current_app.config["LIBREFM_API_KEY"],
}
@@ -161,12 +143,14 @@ def _get_service_or_raise_404(name: str, include_mb=False, exclude_apple=False)
return CritiqueBrainzService()
elif service == ExternalServiceType.SOUNDCLOUD:
return SoundCloudService()
+ elif service == ExternalServiceType.LASTFM:
+ return LastfmService()
elif not exclude_apple and service == ExternalServiceType.APPLE:
return AppleService()
elif include_mb and service == ExternalServiceType.MUSICBRAINZ:
return MusicBrainzService()
except KeyError:
- raise NotFound("Service %s is invalid." % name)
+ raise NotFound("Service %s is invalid." % (name,))
@settings_bp.route('/music-services/details/', methods=['POST'])
@@ -198,13 +182,24 @@ def music_services_details():
apple_user = apple_service.get_user(current_user.id)
current_apple_permissions = "listen" if apple_user and apple_user["refresh_token"] else "disable"
+ lastfm_service = LastfmService()
+ lastfm_user = lastfm_service.get_user(current_user.id)
+ current_lastfm_permissions = "import" if lastfm_user else "disable"
+
data = {
"current_spotify_permissions": current_spotify_permissions,
"current_critiquebrainz_permissions": current_critiquebrainz_permissions,
"current_soundcloud_permissions": current_soundcloud_permissions,
"current_apple_permissions": current_apple_permissions,
+ "current_lastfm_permissions": current_lastfm_permissions,
}
+ if lastfm_user:
+ data["current_lastfm_settings"] = {
+ "external_user_id": lastfm_user["external_user_id"],
+ "latest_listened_at": lastfm_user["latest_listened_at"],
+ }
+
return jsonify(data)
@@ -242,6 +237,34 @@ def refresh_service_token(service_name: str):
return jsonify({"access_token": user["access_token"]})
+@settings_bp.route('/music-services//connect/', methods=['POST'])
+@api_login_required
+def music_services_connect(service_name: str):
+ """ Connect last.fm/libre.fm account to ListenBrainz user. """
+ # TODO: add support for libre.fm
+ if service_name.lower() != "lastfm":
+ raise APINotFound("Service %s is invalid." % (service_name,))
+
+ data = request.json
+ if "external_user_id" not in data:
+ raise APIBadRequest("Missing 'external_user_id' in request.")
+
+ latest_listened_at = None
+ if data.get("latest_listened_at") is not None:
+ try:
+ latest_listened_at = datetime.fromisoformat(data["latest_listened_at"])
+ except (ValueError, TypeError):
+ raise APIBadRequest(f"Value of latest_listened_at '{data['latest_listened_at']} is invalid.")
+
+ # TODO: make last.fm start import timestamp configurable
+ service = LastfmService()
+ service.add_new_user(current_user.id, {
+ "external_user_id": data["external_user_id"],
+ "latest_listened_at": latest_listened_at,
+ })
+ return jsonify({"success": True})
+
+
@settings_bp.route('/music-services//disconnect/', methods=['POST'])
@api_login_required
def music_services_disconnect(service_name: str):
diff --git a/listenbrainz/webserver/views/test/test_settings.py b/listenbrainz/webserver/views/test/test_settings.py
index 3d29035813..68e0dbf5d3 100644
--- a/listenbrainz/webserver/views/test/test_settings.py
+++ b/listenbrainz/webserver/views/test/test_settings.py
@@ -1,3 +1,5 @@
+from datetime import datetime, timezone
+
import requests_mock
import spotipy
@@ -32,17 +34,28 @@ def test_settings_view(self):
self.assertIn(self.user['auth_token'], response.data.decode('utf-8'))
def test_reset_import_timestamp(self):
- val = int(time.time())
- listens_importer.update_latest_listened_at(self.db_conn, self.user['id'], ExternalServiceType.LASTFM, val)
self.temporary_login(self.user['login_id'])
- response = self.client.get(self.custom_url_for('settings.index', path='resetlatestimportts'))
- self.assertTemplateUsed('index.html')
+ response = self.client.post(
+ self.custom_url_for('settings.music_services_connect', service_name='lastfm'),
+ json={"external_user_id": "lucifer"}
+ )
+ self.assert200(response)
+ users = listens_importer.get_active_users_to_process(self.db_conn, ExternalServiceType.LASTFM)
+ self.assertEqual(len(users), 1)
+ self.assertEqual(users[0]["external_user_id"], "lucifer")
+ self.assertEqual(users[0]["latest_listened_at"], None)
+
+ dt_now = datetime.now(tz=timezone.utc)
+ response = self.client.post(
+ self.custom_url_for('settings.music_services_connect', service_name='lastfm'),
+ json={"external_user_id": "lucifer", "latest_listened_at": dt_now.isoformat()}
+ )
self.assert200(response)
- response = self.client.post(self.custom_url_for('settings.reset_latest_import_timestamp'))
- self.assertDictEqual(response.json, {'success': True})
- ts = listens_importer.get_latest_listened_at(self.db_conn, self.user['id'], ExternalServiceType.LASTFM)
- self.assertEqual(int(ts.strftime('%s')), 0)
+ users = listens_importer.get_active_users_to_process(self.db_conn, ExternalServiceType.LASTFM)
+ self.assertEqual(len(users), 1)
+ self.assertEqual(users[0]["external_user_id"], "lucifer")
+ self.assertEqual(users[0]["latest_listened_at"], dt_now)
def test_user_info_not_logged_in(self):
"""Tests user info view when not logged in"""