Merge branch 'sictiru' of github.com:samuelclay/NewsBlur into sictiru

Maintenance.md

@@ -137,6 +137,9 @@ Provision a new mongo server, replicate the data, take newsblur down for mainten
    make plan
    make apply
    make firewall
+   ./utils/ssh.sh db-mongo-primary1
+      docker exec -it mongo mongo
+      mongo> rs.add("db-mongo-primary1.node.nyc1.consul:27017")
    # Wait for mongo to synbc, takes 4-5 hours
    make celery_stop
    make maintenance_on

README.md

@@ -12,6 +12,8 @@
 <a href="https://play.google.com/store/apps/details?id=com.newsblur" target="_blank">
 <img src="https://play.google.com/intl/en_us/badges/images/generic/en-play-badge.png" alt="Get it on Google Play" height="80"/></a>
 
+&nbsp;&nbsp;&nbsp;<a href="https://apps.apple.com/us/app/newsblur/id463981119"><img src="https://tools.applemediaservices.com/api/badges/download-on-the-app-store/black/en-us?size=250x83" alt="Download on the Apple App Store" height="55"></a>
+
 ## Features
 
  1. Shows the original site (you have to see it to believe it).

ansible/playbooks/deploy_app.yml

@@ -78,6 +78,7 @@
         overwrite: different
         aws_access_key: "{{ lookup('ini', 'aws_access_key_id section=default file=/srv/secrets-newsblur/keys/aws.s3.token') }}"
         aws_secret_key: "{{ lookup('ini', 'aws_secret_access_key section=default file=/srv/secrets-newsblur/keys/aws.s3.token') }}"
+        endpoint_url: "https://s3.us-east-1.amazonaws.com"
       tags:
         - never
         - static
@@ -93,6 +94,7 @@
         overwrite: different
         aws_access_key: "{{ lookup('ini', 'aws_access_key_id section=default file=/srv/secrets-newsblur/keys/aws.s3.token') }}"
         aws_secret_key: "{{ lookup('ini', 'aws_secret_access_key section=default file=/srv/secrets-newsblur/keys/aws.s3.token') }}"
+        endpoint_url: "https://s3.us-east-1.amazonaws.com"
       tags:
         - never
         - static

ansible/roles/backups/tasks/main.yml

@@ -103,7 +103,18 @@
         become: yes
         command:
           docker run --rm --name=pg_basebackup --network=host -e POSTGRES_PASSWORD=newsblur -v /srv/newsblur/docker/volumes/postgres/data:/var/lib/postgresql/data postgres:13 pg_basebackup -h db-postgres.service.nyc1.consul -p 5432 -U newsblur -D /var/lib/postgresql/data -Fp -R -Xs -P -c fast
-
+      - name: Create Postgres docker volumes with correct permissions
+        become: yes
+        file:
+          path: "{{ item  }}"
+          state: directory
+          recurse: yes
+          owner: "{{ ansible_effective_user_id|int }}"
+          group: "{{ ansible_effective_group_id|int }}"  
+        with_items:
+          - /srv/newsblur/docker/volumes/postgres/archive
+          - /srv/newsblur/docker/volumes/postgres/backups
+          - /srv/newsblur/docker/volumes/postgres/data
       - name: start postgresql
         become: yes
         command:

ansible/roles/postgres/templates/consul_service.json

@@ -1,6 +1,6 @@
 {
     "service": {
-        {% if inventory_hostname.startswith('db-postgres3') %}
+        {% if inventory_hostname.startswith('db-postgres2') %}
         "name": "db-postgres",
         {% else %}
         "name": "db-postgres-secondary",

ansible/roles/redis/tasks/main.yml

@@ -34,6 +34,7 @@
     dest: /srv/newsblur/docker/redis/redis_replica.conf
   notify: restart redis
   register: updated_config
+  when: "'db-redis-story1' in inventory_hostname"
 
 - name: Create Redis docker volume with correct permissions
   file:
@@ -48,6 +49,7 @@
   docker_container:
     name: redis
     image: redis:7
+    pull: true
     state: started
     command: /usr/local/etc/redis/redis_server.conf
     container_default_behavior: no_defaults

apps/api/views.py

@@ -514,5 +514,5 @@ def ip_addresses(request):
     import digitalocean
     doapi = digitalocean.Manager(token=settings.DO_TOKEN_API_IPADDRESSES)
     droplets = doapi.get_all_droplets()
-    addresses = '\n'.join([d.ip_address for d in droplets])
+    addresses = '\n'.join([d.ip_address for d in droplets if any(name in d.name for name in ['task', 'staging', 'app', 'node'])])
     return HttpResponse(addresses, content_type='text/plain')

apps/notifications/models.py

@@ -259,7 +259,7 @@ def replace_with_newlines(element):
         # else:
         #     subtitle = html.unescape(story['story_title'])
         #     body = replace_with_newlines(soup)
-        body = truncate_chars(body.strip(), 600)
+        body = truncate_chars(body.strip(), 3600)
         if not body:
             body = " "
 
@@ -306,13 +306,15 @@ def send_ios(self, story, user, usersub):
 
         tokens = MUserNotificationTokens.get_tokens_for_user(self.user_id)
         # To update APNS:
-        # 1. Create certificate signing requeswt in Keychain Access
+        # 1. Create certificate signing request in Keychain Access
         # 2. Upload to https://developer.apple.com/account/resources/certificates/list
         # 3. Download to secrets/certificates/ios/aps.cer
-        # 4. Open in Keychain Access and export as aps.p12
-        # 4. Export private key as aps_key.p12 WITH A PASSPHRASE (removed later)
-        # 5. openssl pkcs12 -in aps.p12 -out aps.pem -nodes -clcerts -nokeys
-        # 6. openssl pkcs12 -clcerts -nokeys -out aps.pem -in aps.p12
+        # 4. Open in Keychain Access:
+        #    - export "Apple Push Service: com.newsblur.NewsBlur" as aps.p12 (or just use aps.cer in #5)
+        #    - export private key as aps_key.p12 WITH A PASSPHRASE (removed later)
+        # 5. openssl x509 -in aps.cer -inform DER -out aps.pem -outform PEM
+        # 6. openssl pkcs12 -nocerts -out aps_key.pem -in aps_key.p12
+        # 7. openssl rsa -out aps_key.noenc.pem -in aps_key.pem
         # 7. cat aps.pem aps_key.noenc.pem > aps.p12.pem
         # 8. Verify: openssl s_client -connect gateway.push.apple.com:2195 -cert aps.p12.pem
         # 9. Deploy: aps -l work -t apns,repo,celery
@@ -368,6 +370,17 @@ def send_android(self, story):
     def send_email(self, story, usersub):
         if not self.is_email:
             return
+
+        # Increment the daily email counter for this user
+        r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL)
+        emails_sent_date_key = f"emails_sent:{datetime.datetime.now().strftime('%Y%m%d')}"
+        r.hincrby(emails_sent_date_key, usersub.user_id, 1)
+        r.expire(emails_sent_date_key, 60 * 60 * 24)  # Keep for a day
+        count = int(r.hget(emails_sent_date_key, usersub.user_id) or 0)
+        if count > settings.MAX_EMAILS_SENT_PER_DAY_PER_USER:
+            logging.user(usersub.user, "~BMSent too many email Story notifications by email: ~FR~SB%s~SN~FR emails" % (count))
+            return
+
         feed = usersub.feed
         story_content = self.sanitize_story(story['story_content'])
 

apps/profile/models.py

@@ -20,6 +20,7 @@
 from django.contrib.auth.models import User
 from django.contrib.sites.models import Site
 from django.core.mail import EmailMultiAlternatives
+from django.core.mail import mail_admins
 from django.urls import reverse
 from django.template.loader import render_to_string
 from apps.rss_feeds.models import Feed, MStory, MStarredStory
@@ -232,6 +233,13 @@ def activate_premium(self, never_expire=False):
         from apps.profile.tasks import EmailNewPremium
 
         EmailNewPremium.delay(user_id=self.user.pk)
+
+        subs = UserSubscription.objects.filter(user=self.user)
+        if subs.count() > 5000:
+            logging.user(self.user, "~FR~SK~FW~SBWARNING! ~FR%s subscriptions~SN!" % (subs.count()))
+            mail_admins(f"WARNING! {self.user.username} has {subs.count()} subscriptions", 
+                        f"{self.user.username} has {subs.count()} subscriptions and just upgraded to premium. They'll need a refund: {self.user.profile.paypal_sub_id} {self.user.profile.stripe_id} {self.user.email}")
+            return False
 
         was_premium = self.is_premium
         self.is_premium = True
@@ -242,7 +250,6 @@ def activate_premium(self, never_expire=False):
         self.user.save()
 
         # Only auto-enable every feed if a free user is moving to premium
-        subs = UserSubscription.objects.filter(user=self.user)
         if not was_premium:
             for sub in subs:
                 if sub.active: continue
@@ -276,6 +283,13 @@ def activate_premium(self, never_expire=False):
     def activate_archive(self, never_expire=False):
         UserSubscription.schedule_fetch_archive_feeds_for_user(self.user.pk)
 
+        subs = UserSubscription.objects.filter(user=self.user)
+        if subs.count() > 2000:
+            logging.user(self.user, "~FR~SK~FW~SBWARNING! ~FR%s subscriptions~SN!" % (subs.count()))
+            mail_admins(f"WARNING! {self.user.username} has {subs.count()} subscriptions", 
+                        f"{self.user.username} has {subs.count()} subscriptions and just upgraded to archive. They'll need a refund: {self.user.profile.paypal_sub_id} {self.user.profile.stripe_id} {self.user.email}")
+            return False
+
         was_premium = self.is_premium
         was_archive = self.is_archive
         was_pro = self.is_pro
@@ -286,7 +300,6 @@ def activate_archive(self, never_expire=False):
         self.user.save()
 
         # Only auto-enable every feed if a free user is moving to premium
-        subs = UserSubscription.objects.filter(user=self.user)
         if not was_premium:
             for sub in subs:
                 if sub.active: continue
@@ -324,6 +337,13 @@ def activate_pro(self, never_expire=False):
 
         EmailNewPremiumPro.delay(user_id=self.user.pk)
 
+        subs = UserSubscription.objects.filter(user=self.user)
+        if subs.count() > 1000:
+            logging.user(self.user, "~FR~SK~FW~SBWARNING! ~FR%s subscriptions~SN!" % (subs.count()))
+            mail_admins(f"WARNING! {self.user.username} has {subs.count()} subscriptions", 
+                        f"{self.user.username} has {subs.count()} subscriptions and just upgraded to pro. They'll need a refund: {self.user.profile.paypal_sub_id} {self.user.profile.stripe_id} {self.user.email}")
+            return False
+
         was_premium = self.is_premium
         was_archive = self.is_archive
         was_pro = self.is_pro
@@ -335,7 +355,6 @@ def activate_pro(self, never_expire=False):
         self.user.save()
 
         # Only auto-enable every feed if a free user is moving to premium
-        subs = UserSubscription.objects.filter(user=self.user)
         if not was_premium:
             for sub in subs:
                 if sub.active: continue

apps/rss_feeds/models.py

@@ -2620,7 +2620,7 @@ class MStory(mongo.Document):
 
     def __str__(self):
         content = self.story_content_z if self.story_content_z else ""
-        return f"{self.story_hash}: {self.story_title[:20]} ({len(self.story_content_z)} bytes)"
+        return f"{self.story_hash}: {self.story_title[:20]} ({len(self.story_content_z) if self.story_content_z else 0} bytes)"
 
     @property
     def guid_hash(self):

apps/social/views.py

@@ -1226,7 +1226,7 @@ def find_friends(request):
         if results:
             email = results.group(0)
             profiles = MSocialProfile.objects.filter(email__iexact=email)[:limit]
-    if query.isdigit() and request.user.is_staff:
+    if not profiles and query.isdigit() and request.user.is_staff:
         profiles = MSocialProfile.objects.filter(user_id=int(query))[:limit]
     if not profiles:
         profiles = MSocialProfile.objects.filter(username__iexact=query)[:limit]

clients/android/fastlane/metadata/android/en-US/full_description.txt

@@ -0,0 +1,21 @@
+NewsBlur is a personal <i>news reader<i> bringing people together to talk about the world. Share stories, read your feed subscriptions, and talk to like minded people with this free service. A new sound of an old instrument.
+
+Google Reader has left us and we are working hard to be your replacement. We have many fixes and enhancements planned for all our NewsBlur offerings so stay tuned.
+
+<b>Features:<b>
+<ul>
+    <li>Synchronizes with NewsBlur servers so keep your stories and read/unread status consistent<li>
+    <li>Read all of your news subscriptions (RSS feeds)<li>
+    <li>Read while offline and re-sync read and saved stories when online again.<li>
+    <li>Share and comment on the news with friends<li>
+    <li>Multiple device support (Web, Android, iOS)<li>
+    <li>Meet new people who share the same interests<li>
+    <li>Collapsible folders<li>
+    <li>Save stories<li>
+    <li>Dark mode for night time reading<li>
+    <li>Multiple viewing options for stories<li>
+    <li>Training your feeds - Hide the stories you don't like and highlight the stories you do.<li>
+    <li>River of news - show stories from several different feeds in a single view<li>
+<ul>
+
+https://www.newsblur.com/

clients/android/fastlane/metadata/android/en-US/short_description.txt

@@ -0,0 +1 @@
+Powerful & customizable personal news reader.

clients/android/fastlane/metadata/android/en-US/title.txt

@@ -0,0 +1 @@
+NewsBlur - News reader

clients/ios/Classes/BaseViewController.h

@@ -20,5 +20,10 @@
 - (void)tableView:(UITableView *)tableView selectRowAtIndexPath:(NSIndexPath *)indexPath animated:(BOOL)animated scrollPosition:(UITableViewScrollPosition)scrollPosition;
 - (void)tableView:(UITableView *)tableView deselectRowAtIndexPath:(NSIndexPath *)indexPath animated:(BOOL)animated;
 
+- (void)collectionView:(UICollectionView *)collectionView redisplayCellAtIndexPath:(NSIndexPath *)indexPath;
+- (void)collectionView:(UICollectionView *)collectionView selectItemAtIndexPath:(NSIndexPath *)indexPath animated:(BOOL)animated;
+- (void)collectionView:(UICollectionView *)collectionView selectItemAtIndexPath:(NSIndexPath *)indexPath animated:(BOOL)animated scrollPosition:(UICollectionViewScrollPosition)scrollPosition;
+- (void)collectionView:(UICollectionView *)collectionView deselectItemAtIndexPath:(NSIndexPath *)indexPath animated:(BOOL)animated;
+
 @end
 

clients/ios/Classes/BaseViewController.m

@@ -100,6 +100,24 @@ - (void)tableView:(UITableView *)tableView deselectRowAtIndexPath:(NSIndexPath *
     [self tableView:tableView redisplayCellAtIndexPath:indexPath];
 }
 
+- (void)collectionView:(UICollectionView *)collectionView redisplayCellAtIndexPath:(NSIndexPath *)indexPath {
+    [[collectionView cellForItemAtIndexPath:indexPath] setNeedsDisplay];
+}
+
+- (void)collectionView:(UICollectionView *)collectionView selectItemAtIndexPath:(NSIndexPath *)indexPath animated:(BOOL)animated {
+    [self collectionView:collectionView selectItemAtIndexPath:indexPath animated:animated scrollPosition:UICollectionViewScrollPositionNone];
+}
+
+- (void)collectionView:(UICollectionView *)collectionView selectItemAtIndexPath:(NSIndexPath *)indexPath animated:(BOOL)animated scrollPosition:(UICollectionViewScrollPosition)scrollPosition {
+    [collectionView selectItemAtIndexPath:indexPath animated:animated scrollPosition:scrollPosition];
+    [self collectionView:collectionView redisplayCellAtIndexPath:indexPath];
+}
+
+- (void)collectionView:(UICollectionView *)collectionView deselectItemAtIndexPath:(NSIndexPath *)indexPath animated:(BOOL)animated {
+    [collectionView deselectItemAtIndexPath:indexPath animated:animated];
+    [self collectionView:collectionView redisplayCellAtIndexPath:indexPath];
+}
+
 #pragma mark -
 #pragma mark Keyboard support
 - (void)addKeyCommandWithInput:(NSString *)input modifierFlags:(UIKeyModifierFlags)modifierFlags action:(SEL)action discoverabilityTitle:(NSString *)discoverabilityTitle {
@@ -128,8 +146,10 @@ - (void)addCancelKeyCommandWithAction:(SEL)action discoverabilityTitle:(NSString
 - (void) viewDidLoad {
 	[super viewDidLoad];
 
+    BOOL isDark = [NewsBlurAppDelegate sharedAppDelegate].window.windowScene.traitCollection.userInterfaceStyle == UIUserInterfaceStyleDark;
+
     [[ThemeManager themeManager] addThemeGestureRecognizerToView:self.view];
-    [[ThemeManager themeManager] systemAppearanceDidChange:self.traitCollection.userInterfaceStyle == UIUserInterfaceStyleDark];
+    [[ThemeManager themeManager] systemAppearanceDidChange:isDark];
 }
 
 - (void)viewWillTransitionToSize:(CGSize)size withTransitionCoordinator:(id<UIViewControllerTransitionCoordinator>)coordinator {
@@ -144,9 +164,9 @@ - (void)viewWillTransitionToSize:(CGSize)size withTransitionCoordinator:(id<UIVi
 - (void)traitCollectionDidChange:(UITraitCollection *)previousTraitCollection {
     [super traitCollectionDidChange:previousTraitCollection];
 
-    if ([previousTraitCollection hasDifferentColorAppearanceComparedToTraitCollection:self.traitCollection]) {
-        [[ThemeManager themeManager] systemAppearanceDidChange:self.traitCollection.userInterfaceStyle == UIUserInterfaceStyleDark];
-    }
+    BOOL isDark = [NewsBlurAppDelegate sharedAppDelegate].window.windowScene.traitCollection.userInterfaceStyle == UIUserInterfaceStyleDark;
+
+    [[ThemeManager themeManager] systemAppearanceDidChange:isDark];
 }
 
 - (UIStatusBarStyle)preferredStatusBarStyle {