From 53dcb3e2c6e14ee0da081205e041604abd52ca90 Mon Sep 17 00:00:00 2001
From: Andy <88590076+AAndyProgram@users.noreply.github.com>
Date: Sun, 2 Jun 2024 01:19:41 +0300
Subject: [PATCH] 2024.6.2.0

YT
Add 'FileAddDateToFileName', 'FileAddDateToFileName_VideoForm' and 'FileAddDateToFileName_VideoList' properties
Update 'YouTubeMediaContainerBase', 'VideoOptionsForm' and 'MediaItem' (new options)

SCrawler
API.Instagram: update settings values
API.Reddit: fix 'ReparseMissing' function (remove bearer token)
---
 SCrawler.YouTube/Base/Structures.vb                 |  6 ++++++
 SCrawler.YouTube/Base/YouTubeSettings.vb            | 13 +++++++++++--
 SCrawler.YouTube/Controls/VideoOptionsForm.vb       |  2 +-
 SCrawler.YouTube/Downloader/MediaItem.vb            |  4 ++--
 .../Objects/YouTubeMediaContainerBase.vb            | 10 ++++++++++
 SCrawler/API/Instagram/SiteSettings.vb              |  5 +++++
 SCrawler/API/Reddit/UserData.vb                     | 11 ++++++++---
 7 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/SCrawler.YouTube/Base/Structures.vb b/SCrawler.YouTube/Base/Structures.vb
index 4e2cadc..47ba37f 100644
--- a/SCrawler.YouTube/Base/Structures.vb
+++ b/SCrawler.YouTube/Base/Structures.vb
@@ -78,6 +78,12 @@ Namespace API.YouTube.Base
         https = 1
         m3u8 = 2
     End Enum
+    <Editor(GetType(EnumDropDownEditor), GetType(UITypeEditor))>
+    Public Enum FileDateMode As Integer
+        None = 0
+        Before = 1
+        After = 2
+    End Enum
     Public Structure MediaObject : Implements IIndexable, IComparable(Of MediaObject)
         Public Type As Plugin.UserMediaTypes
         Public ID As String
diff --git a/SCrawler.YouTube/Base/YouTubeSettings.vb b/SCrawler.YouTube/Base/YouTubeSettings.vb
index c715aaa..912c5bf 100644
--- a/SCrawler.YouTube/Base/YouTubeSettings.vb
+++ b/SCrawler.YouTube/Base/YouTubeSettings.vb
@@ -249,9 +249,18 @@ Namespace API.YouTube.Base
         <Browsable(True), GridVisible(False), XMLVN({"Defaults"}), Category("Defaults"), DisplayName("Program description"),
             Description("Add some additional info to the program info if you need")>
         Friend ReadOnly Property ProgramDescription As XMLValue(Of String)
-        <Browsable(True), GridVisible(False), XMLVN({"Defaults"}, "%"""), Category("Defaults"), DisplayName("Remove characters"),
+        <Browsable(True), GridVisible, XMLVN({"Defaults"}, "%"""), Category("Defaults"), DisplayName("Remove characters"),
             Description("Remove specific characters from a file name")>
-        Friend ReadOnly Property FileRemoveCharacters As XMLValue(Of String)
+        Public ReadOnly Property FileRemoveCharacters As XMLValue(Of String)
+        <Browsable(True), GridVisible, XMLVN({"Defaults"}, FileDateMode.None), Category("Defaults"), DisplayName("Add date to file name"),
+            Description("Add the video upload date before/after the file name")>
+        Public ReadOnly Property FileAddDateToFileName As XMLValue(Of FileDateMode)
+        <Browsable(True), GridVisible, XMLVN({"Defaults"}), Category("Defaults"), DisplayName("Add date to title: video form"),
+            Description("Add video upload date before video title (visual only) in the video form")>
+        Public ReadOnly Property FileAddDateToFileName_VideoForm As XMLValue(Of Boolean)
+        <Browsable(True), GridVisible, XMLVN({"Defaults"}), Category("Defaults"), DisplayName("Add date to title: video list"),
+            Description("Add video upload date before video title (visual only) in the video list")>
+        Public ReadOnly Property FileAddDateToFileName_VideoList As XMLValue(Of Boolean)
 #End Region
 #Region "Defaults ChannelsDownload"
         <Browsable(True), GridVisible, XMLVN({"Defaults", "Channels"}), Category("Defaults"), DisplayName("Default download tabs for channels"),
diff --git a/SCrawler.YouTube/Controls/VideoOptionsForm.vb b/SCrawler.YouTube/Controls/VideoOptionsForm.vb
index 766121c..a24283f 100644
--- a/SCrawler.YouTube/Controls/VideoOptionsForm.vb
+++ b/SCrawler.YouTube/Controls/VideoOptionsForm.vb
@@ -121,7 +121,7 @@ Namespace API.YouTube.Controls
                             img = ImageRenderer.GetImage(SFile.GetBytesFromNet(imgUrl, EDP.ReturnValue), EDP.ReturnValue)
                             If Not img Is Nothing Then ICON_VIDEO.Image = img : ICON_VIDEO.InitialImage = img
                         End If
-                        LBL_TITLE.Text = .Title
+                        LBL_TITLE.Text = $"{If(MyYouTubeSettings.FileAddDateToFileName_VideoForm.Value, $"[{ .DateAdded:yyyy-MM-dd}] ", String.Empty)}{ .Title}"
                         LBL_TIME.Text = AConvert(Of String)(.Duration, TimeToStringProvider, String.Empty)
                         TP_HEADER_INFO_2.ColumnStyles(1).Width = MeasureTextDefault(LBL_TIME.Text, LBL_TIME.Font).Width + PaddingE.GetOf({LBL_TIME}).Horizontal
                         TP_HEADER_INFO_2.Refresh()
diff --git a/SCrawler.YouTube/Downloader/MediaItem.vb b/SCrawler.YouTube/Downloader/MediaItem.vb
index 7d9237f..b502cdb 100644
--- a/SCrawler.YouTube/Downloader/MediaItem.vb
+++ b/SCrawler.YouTube/Downloader/MediaItem.vb
@@ -133,7 +133,7 @@ Namespace DownloadObjects.STDownloader
 
                 ICON_SITE.Image = .SiteIcon
                 LBL_TIME.Text = AConvert(Of String)(.Duration, TimeToStringProvider, String.Empty)
-                LBL_TITLE.Text = .ToString(True)
+                LBL_TITLE.Text = $"{If(MyYouTubeSettings.FileAddDateToFileName_VideoList.Value, $"[{ .DateAdded:yyyy-MM-dd}] ", String.Empty)}{ .ToString(True)}"
                 If Not .SiteKey = YouTubeSiteKey And .ContentType = Plugin.UserMediaTypes.Picture Then
                     LBL_INFO.Text = .File.Extension.StringToUpper
                 ElseIf Not .IsMusic And Not (.MediaType = Plugin.UserMediaTypes.Audio Or .MediaType = Plugin.UserMediaTypes.AudioPre) Then
@@ -221,7 +221,7 @@ Namespace DownloadObjects.STDownloader
                     t = 0
                 End If
 
-                LBL_TITLE.Text = MyContainer.ToString(True)
+                LBL_TITLE.Text = $"{If(MyYouTubeSettings.FileAddDateToFileName_VideoList.Value, $"[{ .DateAdded:yyyy-MM-dd}] ", String.Empty)}{ .ToString(True)}"
 
                 If Not .SiteKey = YouTubeSiteKey Then BTT_VIEW_SETTINGS.Visible = False
 
diff --git a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb
index ad6a361..e406d8d 100644
--- a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb
+++ b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb
@@ -656,6 +656,15 @@ Namespace API.YouTube.Objects
                 End If
             End Set
         End Property
+        Friend Sub FileDateUpdate()
+            Dim n$ = _File.Name.StringTrim
+            Dim s$ = IIf(n.IsEmptyString, String.Empty, " ")
+            Select Case MyYouTubeSettings.FileAddDateToFileName.Value
+                Case FileDateMode.Before : n = $"[{DateAdded:yyyy-MM-dd}]{s}{n}"
+                Case FileDateMode.After : n = $"{n}{s}[{DateAdded:yyyy-MM-dd}]"
+            End Select
+            _File.Name = n
+        End Sub
         Public Property FileSettings As SFile
         Private Property IUserMedia_File As String Implements IUserMedia.File
             Get
@@ -1582,6 +1591,7 @@ Namespace API.YouTube.Objects
                             If tValue.HasValue Then Duration = TimeSpan.FromSeconds(tValue.Value)
                         End If
                         DateAdded = AConvert(Of Date)(.Value("release_date").IfNullOrEmpty(.Value("upload_date")), DateAddedProvider, New Date)
+                        If Not IsMusic Then FileDateUpdate()
 
                         ParseFormats(.Self)
 
diff --git a/SCrawler/API/Instagram/SiteSettings.vb b/SCrawler/API/Instagram/SiteSettings.vb
index 4346d6b..812c49d 100644
--- a/SCrawler/API/Instagram/SiteSettings.vb
+++ b/SCrawler/API/Instagram/SiteSettings.vb
@@ -446,6 +446,11 @@ Namespace API.Instagram
         Friend Overrides Sub EndInit()
             Try : MyLastRequests.Add(LastDownloadDate.Value, LastRequestsCount.Value) : Catch : End Try
             If Not CBool(HH_IG_WWW_CLAIM_USE.Value) Then Responser.Headers.Remove(Header_IG_WWW_CLAIM)
+            If CInt(SettingsVersion.Value) < 1 Then
+                SettingsVersion.Value = 1
+                HH_IG_WWW_CLAIM_UPDATE_INTERVAL.Value = 59
+                HH_IG_WWW_CLAIM_USE_DEFAULT_ALGO.Value = 0
+            End If
             MyBase.EndInit()
         End Sub
 #End Region
diff --git a/SCrawler/API/Reddit/UserData.vb b/SCrawler/API/Reddit/UserData.vb
index 254d83a..474781e 100644
--- a/SCrawler/API/Reddit/UserData.vb
+++ b/SCrawler/API/Reddit/UserData.vb
@@ -914,19 +914,24 @@ Namespace API.Reddit
                     Dim RedGifsHost As SettingsHost = Settings(RedGifs.RedGifsSiteKey, RedGifsAccount)
                     If RedGifsHost Is Nothing Then RedGifsHost = Settings(RedGifs.RedGifsSiteKey).Default
                     RedGifsResponser = RedGifsHost.Responser.Copy
+                    Dim respNoHeaders As Responser = Responser.Copy
                     Dim m As UserMedia, m2 As UserMedia
-                    Dim r$
+                    Dim r$, url$
                     Dim j As EContainer
                     Dim lastCount%, li%
+                    Dim rv As New ErrorsDescriber(EDP.ReturnValue)
+                    respNoHeaders.Headers.Clear()
                     ProgressPre.ChangeMax(_ContentList.Count)
                     For i% = 0 To _ContentList.Count - 1
                         m = _ContentList(i)
                         ProgressPre.Perform()
                         If m.State = UStates.Missing AndAlso Not m.Post.ID.IsEmptyString Then
                             ThrowAny(Token)
-                            r = Responser.GetResponse($"https://www.reddit.com/comments/{m.Post.ID.Split("_").LastOrDefault}/.json",, EDP.ReturnValue)
+                            url = $"https://www.reddit.com/comments/{m.Post.ID.Split("_").LastOrDefault}/.json"
+                            r = Responser.GetResponse(url,, rv)
+                            If r.IsEmptyString Then r = respNoHeaders.GetResponse(url,, rv)
                             If Not r.IsEmptyString Then
-                                j = JsonDocument.Parse(r, EDP.ReturnValue)
+                                j = JsonDocument.Parse(r, rv)
                                 If Not j Is Nothing Then
                                     If j.Count > 0 Then
                                         lastCount = _TempMediaList.Count