2022.9.13.0

Added video duration to the feed
Added skipping of pinned Instagram posts if they are already downloaded

Changelog.md

@@ -1,3 +1,13 @@
+# 2022.9.13.0
+
+*2022-09-13*
+
+- Added
+  - Video duration to the feed
+- Fixed
+  - (Issue #70) Instagram posts not downloading if there are pinned posts that have already been downloaded
+  - Minor bugs
+
 # 2022.9.10.0
 
 *2022-09-10*

SCrawler/API/Base/UserDataBase.vb

@@ -830,7 +830,7 @@ BlockNullPicture:
                 Else
                     'ReparseMissing(Token)
                 End If
-                _TempMediaList.ListAddList(ContentMissing, LNC)
+                '_TempMediaList.ListAddList(ContentMissing, LNC)
 
                 If _TempMediaList.Count > 0 Then
                     If Not DownloadImages Then _TempMediaList.RemoveAll(Function(m) m.Type = UTypes.GIF Or m.Type = UTypes.Picture)
@@ -845,9 +845,12 @@ BlockNullPicture:
                 _ContentNew.ListAddList(_TempMediaList, LAP.ClearBeforeAdd)
                 DownloadContent(Token)
                 ThrowIfDisposed()
+
                 LatestData.ListAddList(_ContentNew.Where(_downContent), LNC)
+                Dim mcb& = If(ContentMissingExists, _ContentList.LongCount(Function(c) MissingFinder(c)), 0)
                 _ContentList.ListAddList(_ContentNew.Where(Function(c) _downContent(c) Or MissingFinder(c)), LNC)
-                If DownloadedTotal(False) > 0 Or EnvirChanged.Invoke Or _ContentList.Exists(MissingFinder) Then
+                Dim mca& = If(ContentMissingExists, _ContentList.LongCount(Function(c) MissingFinder(c)), 0)
+                If DownloadedTotal(False) > 0 Or EnvirChanged.Invoke Or Not mcb = mca Then
                     If __SaveData Then
                         LastUpdated = Now
                         RunScript()

SCrawler/API/Instagram/UserData.vb

@@ -247,6 +247,7 @@ Namespace API.Instagram
                     Try
                         Dim n As EContainer, nn As EContainer, node As EContainer
                         Dim HasNextPage As Boolean = False
+                        Dim Pinned As Boolean
                         Dim EndCursor$ = String.Empty
                         Dim PostID$ = String.Empty, PostDate$ = String.Empty, SpecFolder$ = String.Empty
                         Dim TaggedCount%
@@ -296,7 +297,7 @@ Namespace API.Instagram
                         RequestsCount += 1
                         ThrowAny(Token)
 
-                        'Data
+                        'Parsing
                         If Not r.IsEmptyString Then
                             Using j As EContainer = JsonDocument.Parse(r).XmlIfNothing
                                 n = j.ItemF(ENode).XmlIfNothing
@@ -321,13 +322,14 @@ Namespace API.Instagram
                                                         End If
                                                     End If
                                                     PostID = node.Value("id")
-                                                    If Not PostID.IsEmptyString And _TempPostsList.Contains(PostID) Then Throw New ExitException(_DownloadComplete)
+                                                    Pinned = CBool(If(node("pinned_for_users")?.Count, 0))
+                                                    If Not PostID.IsEmptyString And _TempPostsList.Contains(PostID) And Not Pinned Then Throw New ExitException(_DownloadComplete)
                                                     _TempPostsList.Add(PostID)
                                                     PostDate = node.Value("taken_at_timestamp")
                                                     If IsSavedPosts Then
                                                         _SavedPostsIDs.Add(PostID)
                                                     Else
-                                                        If Not CheckDatesLimit(PostDate, DateProvider) Then Throw New ExitException(_DownloadComplete)
+                                                        If Not CheckDatesLimit(PostDate, DateProvider) And Not Pinned Then Throw New ExitException(_DownloadComplete)
                                                         ObtainMedia(node, PostID, PostDate, SpecFolder)
                                                     End If
                                                 Next

SCrawler/Download/FeedMedia.vb

@@ -131,6 +131,7 @@ Namespace DownloadObjects
                         End With
                     End If
 
+                    If Not MyVideo Is Nothing Then info &= $" ({MyVideo.VideoLength})"
                     LBL_INFO.Text = info
                     'TT_MAIN.SetToolTip(LBL_INFO, Information)
                     s = New Size(Width, h + TP_MAIN.RowStyles(0).Height + PaddingE.GetOf({TP_MAIN}).Vertical(2))

SCrawler/Download/FeedVideo.Designer.vb

@@ -30,6 +30,7 @@ Namespace DownloadObjects
             Me.BTT_PAUSE = New System.Windows.Forms.Button()
             Me.BTT_STOP = New System.Windows.Forms.Button()
             Me.TR_VOLUME = New System.Windows.Forms.TrackBar()
+            Me.LBL_TIME = New System.Windows.Forms.Label()
             TP_MAIN = New System.Windows.Forms.TableLayoutPanel()
             TP_BUTTONS = New System.Windows.Forms.TableLayoutPanel()
             TP_MAIN.SuspendLayout()
@@ -88,6 +89,7 @@ Namespace DownloadObjects
             TP_BUTTONS.Controls.Add(Me.BTT_PAUSE, 1, 0)
             TP_BUTTONS.Controls.Add(Me.BTT_STOP, 2, 0)
             TP_BUTTONS.Controls.Add(Me.TR_VOLUME, 4, 0)
+            TP_BUTTONS.Controls.Add(Me.LBL_TIME, 3, 0)
             TP_BUTTONS.Dock = System.Windows.Forms.DockStyle.Fill
             TP_BUTTONS.Location = New System.Drawing.Point(1, 133)
             TP_BUTTONS.Margin = New System.Windows.Forms.Padding(1)
@@ -141,6 +143,16 @@ Namespace DownloadObjects
             Me.TR_VOLUME.Size = New System.Drawing.Size(94, 20)
             Me.TR_VOLUME.TabIndex = 3
             '
+            'LBL_TIME
+            '
+            Me.LBL_TIME.AutoSize = True
+            Me.LBL_TIME.Dock = System.Windows.Forms.DockStyle.Fill
+            Me.LBL_TIME.Location = New System.Drawing.Point(78, 0)
+            Me.LBL_TIME.Name = "LBL_TIME"
+            Me.LBL_TIME.Size = New System.Drawing.Size(1, 26)
+            Me.LBL_TIME.TabIndex = 4
+            Me.LBL_TIME.TextAlign = System.Drawing.ContentAlignment.MiddleCenter
+            '
             'FeedVideo
             '
             Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!)
@@ -164,5 +176,6 @@ Namespace DownloadObjects
         Private WithEvents BTT_PAUSE As Button
         Private WithEvents BTT_STOP As Button
         Private WithEvents TR_VOLUME As TrackBar
+        Private WithEvents LBL_TIME As Label
     End Class
 End Namespace

SCrawler/Download/FeedVideo.vb

@@ -18,7 +18,14 @@ Namespace DownloadObjects
                                                     Dim v# = DivideWithZeroChecking(MediaPlayer.Time, MediaPlayer.Length) * 10
                                                     If v > 10 Then TR_POSITION.Value = 10 Else TR_POSITION.Value = v
                                                 End Sub
+        Private ReadOnly TimeChangeLabel As Action = Sub()
+                                                         If MediaPlayer.Time >= 0 Then
+                                                             Dim t As TimeSpan = TimeSpan.FromMilliseconds(MediaPlayer.Time)
+                                                             LBL_TIME.Text = $"{VideoLength}/{t}"
+                                                         End If
+                                                     End Sub
         Private ReadOnly MyImage As ImageRenderer
+        Friend ReadOnly VideoLength As TimeSpan
         Public Sub New()
             InitializeComponent()
         End Sub
@@ -31,6 +38,7 @@ Namespace DownloadObjects
             MediaPlayer = New [Shared].MediaPlayer(New [Shared].Media(New [Shared].LibVLC(enableDebugLogs:=debugLogs), New Uri(File.ToString)))
             MyVideo.MediaPlayer = MediaPlayer
             TR_VOLUME.Value = MediaPlayer.Volume / 10
+            If MediaPlayer.Length >= 0 Then VideoLength = TimeSpan.FromMilliseconds(MediaPlayer.Length)
             If Settings.UseM3U8 Then
                 Dim f As SFile = $"{Settings.CachePath.PathWithSeparator}FeedSnapshots\{File.GetHashCode}.png"
                 If Not f.Exists Then f = FFMPEG.TakeSnapshot(File, f, Settings.FfmpegFile, TimeSpan.FromSeconds(1))
@@ -58,6 +66,7 @@ Namespace DownloadObjects
         End Sub
         Private Sub MediaPlayer_TimeChanged(sender As Object, e As [Shared].MediaPlayerTimeChangedEventArgs) Handles MediaPlayer.TimeChanged
             If TR_POSITION.InvokeRequired Then TR_POSITION.Invoke(TimeChange) Else TimeChange.Invoke
+            If LBL_TIME.InvokeRequired Then LBL_TIME.Invoke(TimeChangeLabel) Else TimeChangeLabel.Invoke
         End Sub
         Private Sub TR_POSITION_MouseUp(sender As Object, e As MouseEventArgs) Handles TR_POSITION.MouseUp
             Try : MediaPlayer.Time = (MediaPlayer.Length / 100) * (TR_POSITION.Value * 10) : Catch : End Try

SCrawler/MainFrame.Designer.vb

@@ -40,6 +40,7 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
         Me.BTT_DELETE_USER = New System.Windows.Forms.ToolStripButton()
         Me.BTT_REFRESH = New System.Windows.Forms.ToolStripButton()
         Me.BTT_SHOW_INFO = New System.Windows.Forms.ToolStripButton()
+        Me.BTT_FEED = New System.Windows.Forms.ToolStripButton()
         Me.BTT_CHANNELS = New System.Windows.Forms.ToolStripButton()
         Me.BTT_DOWN_SAVED = New System.Windows.Forms.ToolStripButton()
         Me.BTT_DOWN_SELECTED = New System.Windows.Forms.ToolStripButton()
@@ -105,7 +106,6 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
         Me.BTT_TRAY_SHOW_HIDE = New System.Windows.Forms.ToolStripMenuItem()
         Me.BTT_TRAY_CLOSE = New System.Windows.Forms.ToolStripMenuItem()
         Me.BTT_TRAY_CLOSE_NO_SCRIPT = New System.Windows.Forms.ToolStripMenuItem()
-        Me.BTT_FEED = New System.Windows.Forms.ToolStripButton()
         SEP_1 = New System.Windows.Forms.ToolStripSeparator()
         SEP_2 = New System.Windows.Forms.ToolStripSeparator()
         CONTEXT_SEP_1 = New System.Windows.Forms.ToolStripSeparator()
@@ -147,7 +147,7 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
         'MENU_SETTINGS_SEP_1
         '
         MENU_SETTINGS_SEP_1.Name = "MENU_SETTINGS_SEP_1"
-        MENU_SETTINGS_SEP_1.Size = New System.Drawing.Size(113, 6)
+        MENU_SETTINGS_SEP_1.Size = New System.Drawing.Size(177, 6)
         '
         'SEP_3
         '
@@ -228,7 +228,7 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
         '
         Me.BTT_SETTINGS.Image = Global.SCrawler.My.Resources.Resources.SettingsPic_16
         Me.BTT_SETTINGS.Name = "BTT_SETTINGS"
-        Me.BTT_SETTINGS.Size = New System.Drawing.Size(116, 22)
+        Me.BTT_SETTINGS.Size = New System.Drawing.Size(180, 22)
         Me.BTT_SETTINGS.Text = "Settings"
         '
         'Toolbar_TOP
@@ -269,12 +269,12 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
         '
         'BTT_REFRESH
         '
-        Me.BTT_REFRESH.AutoToolTip = False
         Me.BTT_REFRESH.Image = Global.SCrawler.My.Resources.Resources.Refresh
         Me.BTT_REFRESH.ImageTransparentColor = System.Drawing.Color.Magenta
         Me.BTT_REFRESH.Name = "BTT_REFRESH"
         Me.BTT_REFRESH.Size = New System.Drawing.Size(66, 22)
         Me.BTT_REFRESH.Text = "Refresh"
+        Me.BTT_REFRESH.ToolTipText = "Refresh user list"
         '
         'BTT_SHOW_INFO
         '
@@ -285,6 +285,15 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
         Me.BTT_SHOW_INFO.Text = "Info"
         Me.BTT_SHOW_INFO.ToolTipText = "Left-click: open the 'Info' form (show download summary)." & Global.Microsoft.VisualBasic.ChrW(13) & Global.Microsoft.VisualBasic.ChrW(10) & "Right click: open the " &
     "'Missing' form (show information about missing posts)."
+        '
+        'BTT_FEED
+        '
+        Me.BTT_FEED.Image = Global.SCrawler.My.Resources.Resources.RSSPic
+        Me.BTT_FEED.ImageTransparentColor = System.Drawing.Color.Magenta
+        Me.BTT_FEED.Name = "BTT_FEED"
+        Me.BTT_FEED.Size = New System.Drawing.Size(52, 22)
+        Me.BTT_FEED.Text = "Feed"
+        Me.BTT_FEED.ToolTipText = "Feed of recently downloaded data"
         '
         'BTT_CHANNELS
         '
@@ -776,14 +785,6 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
         Me.BTT_TRAY_CLOSE_NO_SCRIPT.ToolTipText = "Close the program without executing the script"
         Me.BTT_TRAY_CLOSE_NO_SCRIPT.Visible = False
         '
-        'BTT_FEED
-        '
-        Me.BTT_FEED.Image = Global.SCrawler.My.Resources.Resources.RSSPic
-        Me.BTT_FEED.ImageTransparentColor = System.Drawing.Color.Magenta
-        Me.BTT_FEED.Name = "BTT_FEED"
-        Me.BTT_FEED.Size = New System.Drawing.Size(52, 22)
-        Me.BTT_FEED.Text = "Feed"
-        '
         'MainFrame
         '
         Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!)

SCrawler/MainFrame.vb

@@ -1171,10 +1171,10 @@ ResumeDownloadingOperation:
         Dim a As Action = Sub()
                               Dim i% = LIST_PROFILES.Items.IndexOfKey(Key)
                               If i < 0 Then
-                                  i = Settings.Users.FindIndex(Function(u) u.Key = Key)
+                                  Dim u As IUserData = Settings.GetUser(Key, True)
-                                  If i >= 0 Then
+                                  If Not u Is Nothing Then
-                                      UserListUpdate(Settings.Users(i), True)
+                                      UserListUpdate(u, True)
-                                      i = LIST_PROFILES.Items.IndexOfKey(Key)
+                                      i = LIST_PROFILES.Items.IndexOfKey(u.Key)
                                   End If
                               End If
                               If i >= 0 Then

SCrawler/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2022.9.10.0")>
+<Assembly: AssemblyVersion("2022.9.13.0")>
-<Assembly: AssemblyFileVersion("2022.9.10.0")>
+<Assembly: AssemblyFileVersion("2022.9.13.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/SettingsCLS.vb

@@ -343,7 +343,7 @@ Friend Class SettingsCLS : Implements IDisposable
             _UserListUpdateRequired = True
         End Try
     End Sub
-    Friend Overloads Function GetUser(ByVal User As IUserData) As IUserData
+    Friend Overloads Function GetUser(ByVal User As IUserData, Optional ByVal GetCollection As Boolean = False) As IUserData
         If Users.Count > 0 Then
             Dim uSimple As Predicate(Of IUserData) = Function(u) u.Equals(DirectCast(User, UserDataBase))
             Dim uCol As Predicate(Of IUserData) = Function(ByVal u As IUserData) As Boolean
@@ -360,7 +360,7 @@ Friend Class SettingsCLS : Implements IDisposable
                 If Users(i).IsCollection Then
                     With DirectCast(Users(i), UserDataBind)
                         i = .Collections.FindIndex(uSimple)
-                        If i >= 0 Then Return .Collections(i)
+                        If i >= 0 Then Return If(GetCollection, Users(i), .Collections(i))
                     End With
                 Else
                     Return Users(i)
@@ -369,7 +369,7 @@ Friend Class SettingsCLS : Implements IDisposable
         End If
         Return Nothing
     End Function
-    Friend Overloads Function GetUser(ByVal UserKey As String) As IUserData
+    Friend Overloads Function GetUser(ByVal UserKey As String, Optional ByVal GetCollection As Boolean = False) As IUserData
         If Users.Count > 0 Then
             Dim finder As Predicate(Of IUserData) = Function(u) u.Key = UserKey
             Dim i%, ii%
@@ -379,7 +379,7 @@ Friend Class SettingsCLS : Implements IDisposable
                         With DirectCast(.Self, UserDataBind)
                             If .Count > 0 Then
                                 ii = .Collections.FindIndex(finder)
-                                If ii >= 0 Then Return .Collections(ii)
+                                If ii >= 0 Then Return If(GetCollection, .Self, .Collections(ii))
                             End If
                         End With
                     Else