2023.12.20.0

API.Twitter: simplify JSON error string
Add notification of new log data
Add deleting permanent cache path if empty

SCrawler/API/Twitter/UserData.vb

@@ -364,7 +364,7 @@ Namespace API.Twitter
                 DownloadModelForceApply = False
                 FirstDownloadComplete = True
             Catch jsonNull_ex As ArgumentNullException When jsonArgs.State = WebDocumentEventArgs.States.Error
-                Throw New Plugin.ExitException($"{ToStringForLog()}: No deserialized data found")
+                Throw New Plugin.ExitException("No deserialized data found")
             Catch limit_ex As TwitterLimitException
                 Throw limit_ex
             Catch ex As Exception

SCrawler/Editors/GlobalSettingsForm.Designer.vb

@@ -157,6 +157,7 @@ Namespace Editors
             Me.TXT_FEED_CENTER_IMAGE = New PersonalUtilities.Forms.Controls.TextBoxExtended()
             Me.COLORS_FEED = New SCrawler.Editors.ColorPicker()
             Me.CH_FEED_SHOW_FRIENDLY = New System.Windows.Forms.CheckBox()
+            Me.CH_FEED_SHOW_SPEC_MEDIAITEM = New System.Windows.Forms.CheckBox()
             Me.TXT_YTDLP = New PersonalUtilities.Forms.Controls.TextBoxExtended()
             Me.TXT_FFMPEG = New PersonalUtilities.Forms.Controls.TextBoxExtended()
             Me.TXT_CURL = New PersonalUtilities.Forms.Controls.TextBoxExtended()
@@ -177,7 +178,7 @@ Namespace Editors
             Me.TAB_MAIN = New System.Windows.Forms.TabControl()
             Me.TAB_ENVIR = New System.Windows.Forms.TabPage()
             Me.CONTAINER_MAIN = New System.Windows.Forms.ToolStripContainer()
-            Me.CH_FEED_SHOW_SPEC_MEDIAITEM = New System.Windows.Forms.CheckBox()
+            Me.CH_NOTIFY_LOG = New System.Windows.Forms.CheckBox()
             TP_BASIS = New System.Windows.Forms.TableLayoutPanel()
             TP_IMAGES = New System.Windows.Forms.TableLayoutPanel()
             TP_FILE_NAME = New System.Windows.Forms.TableLayoutPanel()
@@ -1670,6 +1671,17 @@ Namespace Editors
             Me.CH_FEED_SHOW_FRIENDLY.Text = "Show friendly names instead of usernames"
             Me.CH_FEED_SHOW_FRIENDLY.UseVisualStyleBackColor = True
             '
+            'CH_FEED_SHOW_SPEC_MEDIAITEM
+            '
+            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.AutoSize = True
+            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.Dock = System.Windows.Forms.DockStyle.Fill
+            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.Location = New System.Drawing.Point(4, 247)
+            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.Name = "CH_FEED_SHOW_SPEC_MEDIAITEM"
+            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.Size = New System.Drawing.Size(568, 19)
+            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.TabIndex = 9
+            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.Text = "Show special feeds in media items"
+            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.UseVisualStyleBackColor = True
+            '
             'TAB_NOTIFY
             '
             TAB_NOTIFY.Controls.Add(TP_NOTIFY_MAIN)
@@ -1692,10 +1704,12 @@ Namespace Editors
             TP_NOTIFY_MAIN.Controls.Add(Me.CH_NOTIFY_SAVED_POSTS, 0, 5)
             TP_NOTIFY_MAIN.Controls.Add(Me.CH_STD, 0, 6)
             TP_NOTIFY_MAIN.Controls.Add(Me.CH_STD_EVERY, 0, 7)
+            TP_NOTIFY_MAIN.Controls.Add(Me.CH_NOTIFY_LOG, 0, 8)
             TP_NOTIFY_MAIN.Dock = System.Windows.Forms.DockStyle.Fill
             TP_NOTIFY_MAIN.Location = New System.Drawing.Point(0, 0)
             TP_NOTIFY_MAIN.Name = "TP_NOTIFY_MAIN"
-            TP_NOTIFY_MAIN.RowCount = 9
+            TP_NOTIFY_MAIN.RowCount = 10
+            TP_NOTIFY_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
             TP_NOTIFY_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
             TP_NOTIFY_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
             TP_NOTIFY_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
@@ -2140,16 +2154,17 @@ Namespace Editors
             Me.CONTAINER_MAIN.TabIndex = 0
             Me.CONTAINER_MAIN.TopToolStripPanelVisible = False
             '
-            'CH_FEED_SHOW_SPEC_MEDIAITEM
+            'CH_NOTIFY_LOG
             '
-            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.AutoSize = True
+            Me.CH_NOTIFY_LOG.AutoSize = True
-            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.Dock = System.Windows.Forms.DockStyle.Fill
+            Me.CH_NOTIFY_LOG.Dock = System.Windows.Forms.DockStyle.Fill
-            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.Location = New System.Drawing.Point(4, 247)
+            Me.CH_NOTIFY_LOG.Location = New System.Drawing.Point(4, 212)
-            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.Name = "CH_FEED_SHOW_SPEC_MEDIAITEM"
+            Me.CH_NOTIFY_LOG.Name = "CH_NOTIFY_LOG"
-            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.Size = New System.Drawing.Size(568, 19)
+            Me.CH_NOTIFY_LOG.Size = New System.Drawing.Size(568, 19)
-            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.TabIndex = 9
+            Me.CH_NOTIFY_LOG.TabIndex = 8
-            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.Text = "Show special feeds in media items"
+            Me.CH_NOTIFY_LOG.Text = "The log contains new data"
-            Me.CH_FEED_SHOW_SPEC_MEDIAITEM.UseVisualStyleBackColor = True
+            TT_MAIN.SetToolTip(Me.CH_NOTIFY_LOG, "Show a notification when the new data is added to the log.")
+            Me.CH_NOTIFY_LOG.UseVisualStyleBackColor = True
             '
             'GlobalSettingsForm
             '
@@ -2341,5 +2356,6 @@ Namespace Editors
         Private WithEvents CH_STD_YT_CREATE_URL As CheckBox
         Private WithEvents CH_USE_DEF_ACC As CheckBox
         Private WithEvents CH_FEED_SHOW_SPEC_MEDIAITEM As CheckBox
+        Private WithEvents CH_NOTIFY_LOG As CheckBox
     End Class
 End Namespace

SCrawler/Editors/GlobalSettingsForm.vb

@@ -74,6 +74,7 @@ Namespace Editors
                         CH_NOTIFY_SAVED_POSTS.Checked = .ShowNotificationsDownSavedPosts
                         CH_STD.Checked = .ShowNotificationsSTDownloader
                         CH_STD_EVERY.Checked = .ShowNotificationsSTDownloaderEveryDownload
+                        CH_NOTIFY_LOG.Checked = .ShowNotificationsLOG
                         'Defaults
                         CH_SEPARATE_VIDEO_FOLDER.Checked = .SeparateVideoFolder.Value
                         CH_DEF_TEMP.Checked = .DefaultTemporary
@@ -248,6 +249,7 @@ Namespace Editors
                     .ShowNotificationsDownSavedPosts.Value = CH_NOTIFY_SAVED_POSTS.Checked
                     .ShowNotificationsSTDownloader.Value = CH_STD.Checked
                     .ShowNotificationsSTDownloaderEveryDownload.Value = CH_STD_EVERY.Checked
+                    .ShowNotificationsLOG.Value = CH_NOTIFY_LOG.Checked
                     'Defaults
                     .SeparateVideoFolder.Value = CH_SEPARATE_VIDEO_FOLDER.Checked
                     .DefaultTemporary.Value = CH_DEF_TEMP.Checked

SCrawler/MainFrame.vb

@@ -842,7 +842,10 @@ CloseResume:
 #End Region
 #End Region
     Private Sub BTT_LOG_Click(sender As Object, e As EventArgs) Handles BTT_LOG.Click
-        MyMainLOG_ShowForm(Settings.Design,,,, Sub() MainFrameObj.UpdateLogButton())
+        MyMainLOG_ShowForm(Settings.Design,,,, Sub()
+                                                   MainFrameObj.UpdateLogButton()
+                                                   MainFrameObj.LogFormClosed()
+                                               End Sub)
     End Sub
     Private Sub BTT_VERSION_INFO_Click(sender As Object, e As EventArgs) Handles BTT_VERSION_INFO.Click
         CheckVersion(True)

SCrawler/MainFrameObjects.vb

@@ -19,6 +19,11 @@ Friend Class MainFrameObjects : Implements INotificator
         MF = f
         Notificator = New NotificationsManager
         PauseButtons = New DownloadObjects.AutoDownloaderPauseButtons(DownloadObjects.AutoDownloaderPauseButtons.ButtonsPlace.MainFrame)
+        ProgramLogInitialize()
+        With ProgramLog
+            AddHandler ProgramLog.TextAdded, AddressOf ProgramLog_TextAdded
+            AddHandler ProgramLog.TextCleared, AddressOf ProgramLog_TextCleared
+        End With
     End Sub
 #Region "Users"
     Friend Sub FocusUser(ByVal Key As String, Optional ByVal ActivateForm As Boolean = False)
@@ -100,4 +105,16 @@ Friend Class MainFrameObjects : Implements INotificator
         End If
     End Sub
 #End Region
+#Region "LOG events support"
+    Private _LogNotificationsEnabled As Boolean = True
+    Private Sub ProgramLog_TextAdded(ByVal Sender As Object, ByVal e As EventArgs)
+        If _LogNotificationsEnabled Then _LogNotificationsEnabled = False : ShowNotification(NotifyObj.LOG, "There is new data in the log")
+    End Sub
+    Private Sub ProgramLog_TextCleared(ByVal Sender As Object, ByVal e As EventArgs)
+        _LogNotificationsEnabled = True
+    End Sub
+    Friend Sub LogFormClosed()
+        _LogNotificationsEnabled = True
+    End Sub
+#End Region
 End Class

SCrawler/SettingsCLS.vb

@@ -27,6 +27,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
     Friend Const CookieEncryptKey As String = "SCrawlerCookiesEncryptKeyword"
     Private Const EnvironmentPath As String = "Environment\"
     Friend Const CollectionsFolderName As String = "Collections"
+    Private Const PermanentCacheSnapshotsPath As String = "_CacheSnapshots\"
     Friend Const DefaultCmdEncoding As Integer = BatchExecutor.UnicodeEncoding
     Friend ReadOnly Design As XmlFile
     Private ReadOnly MyXML As XmlFile
@@ -113,7 +114,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
     Friend ReadOnly Property CacheSnapshots(ByVal Permanent As Boolean) As CacheKeeper
         Get
             If Permanent Then
-                If _CacheSnapshots Is Nothing Then _CacheSnapshots = New CacheKeeper("_CacheSnapshots\") With {.DeleteCacheOnDispose = False, .DeleteRootOnDispose = False}
+                If _CacheSnapshots Is Nothing Then _CacheSnapshots = New CacheKeeper(PermanentCacheSnapshotsPath) With {.DeleteCacheOnDispose = False, .DeleteRootOnDispose = False}
                 Return _CacheSnapshots
             Else
                 Dim dir As SFile = $"{Cache.RootDirectory.PathWithSeparator}Snapshots\"
@@ -336,6 +337,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
         ShowNotificationsDownSavedPosts = New XMLValue(Of Boolean)("SavedPosts", True, MyXML, n)
         ShowNotificationsSTDownloader = New XMLValue(Of Boolean)("STDownloader", True, MyXML, n)
         ShowNotificationsSTDownloaderEveryDownload = New XMLValue(Of Boolean)("STDownloaderEveryDownload", True, MyXML, n)
+        ShowNotificationsLOG = New XMLValue(Of Boolean)("LOG", True, MyXML, n)
 
         ProgramText = New XMLValue(Of String)("ProgramText",, MyXML)
         ProgramDescription = New XMLValue(Of String)("ProgramDescription",, MyXML)
@@ -672,6 +674,14 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
     Friend Sub DeleteCachePath()
         Reddit.ChannelsCollection.ChannelsPathCache.Delete(SFO.Path, SFODelete.None, EDP.None)
     End Sub
+    Private Sub DeleteCachePathPermanent()
+        Try
+            Dim f As New SFile(PermanentCacheSnapshotsPath)
+            If f.Exists(SFO.Path, False) AndAlso Not SFile.GetFiles(f,, IO.SearchOption.AllDirectories, EDP.ReturnValue).ListExists Then _
+               f.Delete(SFO.Path, SFODelete.DeletePermanently, EDP.None)
+        Catch
+        End Try
+    End Sub
     Friend Overloads Function UserExists(ByVal UserSite As String, ByVal UserID As String) As Boolean
         Dim UserFinderBase As Predicate(Of IUserData) = Function(user) user.Site = UserSite And user.Name = UserID
         Dim UserFinder As Predicate(Of IUserData) = Function(ByVal user As IUserData) As Boolean
@@ -944,6 +954,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
         Channels
         SavedPosts
         STDownloader
+        LOG
     End Enum
     Friend ReadOnly Property ProcessNotification(ByVal Sender As NotificationObjects) As Boolean
         Get
@@ -955,6 +966,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
                     Case NotificationObjects.Channels : Return ShowNotificationsDownChannels
                     Case NotificationObjects.SavedPosts : Return ShowNotificationsDownSavedPosts
                     Case NotificationObjects.STDownloader : Return ShowNotificationsSTDownloader
+                    Case NotificationObjects.LOG : Return ShowNotificationsLOG
                     Case Else : Return True
                 End Select
             Else
@@ -970,6 +982,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
     Friend ReadOnly Property ShowNotificationsDownSavedPosts As XMLValue(Of Boolean)
     Friend ReadOnly Property ShowNotificationsSTDownloader As XMLValue(Of Boolean)
     Friend ReadOnly Property ShowNotificationsSTDownloaderEveryDownload As XMLValue(Of Boolean)
+    Friend ReadOnly Property ShowNotificationsLOG As XMLValue(Of Boolean)
 #End Region
 #Region "Other program properties"
     Friend ReadOnly Property ProgramText As XMLValue(Of String)
@@ -1007,6 +1020,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
                 End If
                 If Not Automation Is Nothing Then Automation.Dispose()
                 Cache.Dispose()
+                DeleteCachePathPermanent()
                 Plugins.Clear()
                 LastCollections.Clear()
                 Users.ListClearDispose