2024.6.6.0

YT
VideoOptionsForm: file path is cleared when the cancel button is clicked (browse button); remove the context menu when the right clicking on browse button; add 'ButtonRC' class

SCrawler
API.OnlyFans: add check config to the SiteSettings; update config; add 'Keydb_Api' property; reset 'LastDateUpdated' when rules change; add support 'prefix/suffix' and 'start/end' to support other rules formats
DownloadFeedForm: update 'BTT_CURR_SESSION_SET_Click' function
TDownloader: update 'FilesLoadLastSession' function

Changelog.md

@@ -1,3 +1,19 @@
+# 2024.6.6.0
+
+*2024-06-06*
+
+**ATTENTION!**
+1. **To support downloading of DRM protected videos (OnlyFans), please update OF-Scraper to version [3.10](https://github.com/datawhores/OF-Scraper/releases/tag/3.10) (download `zip`, not `exe`).**
+2. **If there is a `OFScraperConfigPattern.json` file in the SCrawler settings folder, replace the text of the file with [this text](https://github.com/AAndyProgram/SCrawler/blob/main/SCrawler/API/OnlyFans/OFScraperConfigPattern.json).**
+3. **Set the value to `Dynamic rules` (in the site settings) = `https://raw.githubusercontent.com/Growik/onlyfans-dynamic-rules/main/rules.json`.**
+
+- Added
+  - OnlyFans: new OF-Scraper option (`keydb_api`)
+  - Minor improvements
+- Fixed
+  - OnlyFans: **data is not downloading**
+  - Minor bugs
+
 # 2024.6.4.0
 
 *2024-06-04*

SCrawler.YouTube/Controls/ButtonRC.vb

@@ -0,0 +1,23 @@
+' Copyright (C) Andy https://github.com/AAndyProgram
+' This program is free software: you can redistribute it and/or modify
+' it under the terms of the GNU General Public License as published by
+' the Free Software Foundation, either version 3 of the License, or
+' (at your option) any later version.
+'
+' This program is distributed in the hope that it will be useful,
+' but WITHOUT ANY WARRANTY
+Namespace API.YouTube.Controls
+    Public Class ButtonRC : Inherits Button
+        Private Const WM_CONTEXTMENU As Integer = 123 '&H7B
+        Private Const WM_CANCELMODE As Integer = 31 '&H1F
+        Private Const WM_INITMENUPOPUP As Integer = 279 '&H117
+        Private Const SMTO_NOTIMEOUTIFNOTHUNG As Integer = 8
+        Protected Overrides Sub WndProc(ByRef m As Message)
+            If m.Msg = WM_CONTEXTMENU Or m.Msg = WM_CANCELMODE Or m.Msg = WM_INITMENUPOPUP Or m.Msg = SMTO_NOTIMEOUTIFNOTHUNG Then
+                m.Result = IntPtr.Zero
+            Else
+                MyBase.WndProc(m)
+            End If
+        End Sub
+    End Class
+End Namespace

SCrawler.YouTube/Controls/VideoOptionsForm.Designer.vb

@@ -65,11 +65,11 @@ Namespace API.YouTube.Controls
             Me.LBL_TIME = New System.Windows.Forms.Label()
             Me.LBL_URL = New System.Windows.Forms.LinkLabel()
             Me.TXT_FILE = New PersonalUtilities.Forms.Controls.ComboBoxExtended()
-            Me.BTT_BROWSE = New System.Windows.Forms.Button()
+            Me.BTT_BROWSE = New SCrawler.API.YouTube.Controls.ButtonRC()
             Me.BTT_DOWN = New System.Windows.Forms.Button()
             Me.BTT_CANCEL = New System.Windows.Forms.Button()
             Me.CMB_PLS = New PersonalUtilities.Forms.Controls.ComboBoxExtended()
-            Me.BTT_PLS_BROWSE = New System.Windows.Forms.Button()
+            Me.BTT_PLS_BROWSE = New SCrawler.API.YouTube.Controls.ButtonRC()
             Me.OPT_VIDEO = New System.Windows.Forms.RadioButton()
             Me.OPT_AUDIO = New System.Windows.Forms.RadioButton()
             Me.LBL_AUDIO_CODEC = New System.Windows.Forms.Label()
@@ -912,13 +912,13 @@ Namespace API.YouTube.Controls
         Private WithEvents TXT_SUBS_ADDIT As PersonalUtilities.Forms.Controls.TextBoxExtended
         Private WithEvents TXT_EXTRA_AUDIO_FORMATS As PersonalUtilities.Forms.Controls.TextBoxExtended
         Private WithEvents TXT_FILE As PersonalUtilities.Forms.Controls.ComboBoxExtended
-        Private WithEvents BTT_BROWSE As Button
+        Private WithEvents BTT_BROWSE As SCrawler.API.YouTube.Controls.ButtonRC
         Private WithEvents BTT_DOWN As Button
         Private WithEvents BTT_CANCEL As Button
         Private WithEvents TP_HEADER_INFO_2 As TableLayoutPanel
         Private WithEvents TXT_FPS As PersonalUtilities.Forms.Controls.TextBoxExtended
         Private WithEvents CMB_PLS As PersonalUtilities.Forms.Controls.ComboBoxExtended
-        Private WithEvents BTT_PLS_BROWSE As Button
+        Private WithEvents BTT_PLS_BROWSE As SCrawler.API.YouTube.Controls.ButtonRC
         Private WithEvents TXT_AUDIO_BITRATE As PersonalUtilities.Forms.Controls.TextBoxExtended
     End Class
 End Namespace

SCrawler.YouTube/Controls/VideoOptionsForm.vb

@@ -610,7 +610,7 @@ Namespace API.YouTube.Controls
                                 $"Video|{AvailableVideoFormats.Select(Function(vf) $"*.{vf.ToLower}").ListToString(";")}" &
                                 $"|Audio|{AvailableAudioFormats.Select(Function(af) $"*.{af.ToLower}").ListToString(";")}"
                 f = SFile.SaveAs(f, "Select the destination of the video file",, ext, sPattern, EDP.ReturnValue)
-                f.Extension = ext
+                If Not f.IsEmptyString Then f.Extension = ext
             End If
 #Enable Warning
             f = CleanFileName(f)

SCrawler.YouTube/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.6.4.0")>
+<Assembly: AssemblyVersion("2024.6.6.0")>
-<Assembly: AssemblyFileVersion("2024.6.4.0")>
+<Assembly: AssemblyFileVersion("2024.6.6.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.YouTube/SCrawler.YouTube.vbproj

@@ -115,6 +115,9 @@
   <ItemGroup>
     <Compile Include="Attributes\GridVisibleAttribute.vb" />
     <Compile Include="Base\TableControlsProcessor.vb" />
+    <Compile Include="Controls\ButtonRC.vb">
+      <SubType>Component</SubType>
+    </Compile>
     <Compile Include="Controls\ChannelTabsChooserForm.Designer.vb">
       <DependentUpon>ChannelTabsChooserForm.vb</DependentUpon>
     </Compile>

SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.6.4.0")>
+<Assembly: AssemblyVersion("2024.6.6.0")>
-<Assembly: AssemblyFileVersion("2024.6.4.0")>
+<Assembly: AssemblyFileVersion("2024.6.6.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/API/OnlyFans/Declarations.vb

@@ -11,5 +11,15 @@ Namespace API.OnlyFans
     Friend Module Declarations
         Friend ReadOnly DateProvider As New ADateTime("O")
         Friend ReadOnly RegExPostID As RParams = RParams.DM("(?<=onlyfans\.com/)(\d+)", 0, EDP.ReturnValue)
+        Friend ReadOnly OFScraperConfigPatternFile As SFile = $"{SettingsFolderName}\OFScraperConfigPattern.json"
+        Friend Function CheckOFSConfig() As Boolean
+            If Not OFScraperConfigPatternFile.Exists Then
+                Dim t$ = Text.Encoding.UTF8.GetString(My.Resources.OFResources.OFScraperConfigPattern)
+                TextSaver.SaveTextToFile(t, OFScraperConfigPatternFile, True)
+                Return OFScraperConfigPatternFile.Exists
+            Else
+                Return True
+            End If
+        End Function
     End Module
 End Namespace

SCrawler/API/OnlyFans/OFScraperConfigPattern.json

@@ -38,14 +38,18 @@
         },
         "advanced_options": {
             "code-execution": false,
-            "dynamic-mode-default": "deviint",
+            "dynamic-mode-default": "sneaky",
             "backend": "aio",
             "downloadbars": false,
             "cache-mode": "sqlite",
             "appendlog": true,
             "custom": null,
             "sanitize_text": false,
-            "avatar": true
+            "avatar": true,
+            "custom_values": {
+                "SNEAKY": "https://raw.githubusercontent.com/Growik/onlyfans-dynamic-rules/main/rules.json",
+                "CDRM": "https://old.cdrm-project.com/wv"
+            }
         },
         "responsetype": {
             "timeline": "Posts",

SCrawler/API/OnlyFans/SiteSettings.vb

@@ -133,12 +133,25 @@ Namespace API.OnlyFans
                 End If
             End Get
         End Property
+        <PClonable, PXML("keydb_api")> Private ReadOnly Property Keydb_Api_XML As PropertyValue
+        <PropertyOption(ControlText:="keydb_api", Category:=CAT_OFS)>
+        Friend ReadOnly Property Keydb_Api As PropertyValue
+            Get
+                If Not DefaultInstance Is Nothing Then
+                    Return DirectCast(DefaultInstance, SiteSettings).Keydb_Api_XML
+                Else
+                    Return Keydb_Api_XML
+                End If
+            End Get
+        End Property
 #End Region
 #End Region
 #Region "Initializer"
         Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean)
             MyBase.New("OnlyFans", ".onlyfans.com", AccName, Temp, My.Resources.SiteResources.OnlyFansIcon_32, My.Resources.SiteResources.OnlyFansPic_32)
 
+            CheckOFSConfig()
+
             _AllowUserAgentUpdate = False
 
             With Responser
@@ -192,6 +205,7 @@ Namespace API.OnlyFans
             End If
             OFScraperMP4decrypt_XML = New PropertyValue(String.Empty, GetType(String))
             KeyModeDefault_XML = New PropertyValue(KeyModeDefault_Default)
+            Keydb_Api_XML = New PropertyValue(String.Empty, GetType(String))
 
             UserRegex = RParams.DMS(String.Format(UserRegexDefaultPattern, "onlyfans.com/"), 1, EDP.ReturnValue)
             UrlPatternUser = "https://onlyfans.com/{0}"
@@ -204,8 +218,19 @@ Namespace API.OnlyFans
         End Function
 #End Region
 #Region "Update"
+        Private __UseOldAuthRules As Boolean = True
+        Private __DynamicRules As String = String.Empty
+        Friend Overrides Sub BeginUpdate()
+            __UseOldAuthRules = UseOldAuthRules.Value
+            __DynamicRules = AConvert(Of String)(DynamicRules.Value, String.Empty)
+            MyBase.BeginUpdate()
+        End Sub
         Friend Overrides Sub Update()
-            If _SiteEditorFormOpened Then Responser.Cookies.Changed = False
+            If _SiteEditorFormOpened Then
+                If Not __UseOldAuthRules = CBool(UseOldAuthRules.Value) Or Not AEquals(Of String)(__DynamicRules, DynamicRules.Value) Then _
+                   LastDateUpdated = LastDateUpdated.AddYears(-1)
+                Responser.Cookies.Changed = False
+            End If
             MyBase.Update()
         End Sub
 #End Region

SCrawler/API/OnlyFans/UserData.vb

@@ -549,7 +549,8 @@ Namespace API.OnlyFans
                                          Optional ByVal Round As Integer = 0) As Boolean
             Try
                 If UpdateAuthFile(ForceUpdateAuth) Then
-                    Const nullMsg$ = "The auth parameter is null"
+                    Const nullMsg$ = "The auth parameter(s) is null"
+                    Const formatMidPart$ = ":{0}:{1:x}:"
                     Dim j As EContainer
                     Try
                         j = JsonDocument.Parse(AuthFile.GetText)
@@ -565,8 +566,16 @@ Namespace API.OnlyFans
                     End Try
                     If Not j Is Nothing Then
                         Dim pattern$ = j.Value("format")
-                        If pattern.IsEmptyString Then Throw New ArgumentNullException("format", nullMsg)
+
-                        pattern = pattern.Replace("{}", "{0}").Replace("{:x}", "{1:x}")
+                        If Not pattern.IsEmptyString Then
+                            pattern = pattern.Replace("{}", "{0}").Replace("{:x}", "{1:x}")
+                        ElseIf Not j.Value("prefix").IsEmptyString And Not j.Value("suffix").IsEmptyString Then
+                            pattern = j.Value("prefix") & formatMidPart & j.Value("suffix")
+                        ElseIf Not j.Value("start").IsEmptyString And Not j.Value("end").IsEmptyString Then
+                            pattern = j.Value("start") & formatMidPart & j.Value("end")
+                        Else
+                            Throw New ArgumentNullException("format", nullMsg)
+                        End If
 
                         Dim li%() = j("checksum_indexes").Select(Function(e) CInt(e(0).Value)).ToArray
 
@@ -607,10 +616,14 @@ Namespace API.OnlyFans
                     Dim r$ = GetWebString(If(ACheck(Of String)(MySettings.DynamicRules.Value),
                                              CStr(MySettings.DynamicRules.Value),
                                              IIf(MySettings.UseOldAuthRules.Value, urlOld, urlNew)),, EDP.ReturnValue)
+                    Dim checkFormat As Func(Of EContainer, Boolean) =
+                        Function(jj) Not jj.Value("format").IsEmptyString OrElse
+                                     (Not jj.Value("prefix").IsEmptyString And Not jj.Value("suffix").IsEmptyString) OrElse
+                                     (Not jj.Value("start").IsEmptyString And Not jj.Value("start").IsEmptyString)
                     If Not r.IsEmptyString Then
                         Using j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue)
                             If j.ListExists Then
-                                If Not j.Value("format").IsEmptyString And j("checksum_indexes").ListExists And
+                                If checkFormat(j) And j("checksum_indexes").ListExists And
                                    Not j.Value("static_param").IsEmptyString And Not j.Value("checksum_constant").IsEmptyString Then _
                                    TextSaver.SaveTextToFile(r, AuthFile, True, False, EDP.ThrowException) : MySettings.LastDateUpdated = Now
                             End If
@@ -658,12 +671,9 @@ Namespace API.OnlyFans
                 currentCache.Validate()
                 Dim cacheRoot As SFile = currentCache.NewPath
                 cacheRoot.Exists(SFO.Path, True, EDP.ThrowException)
-                Dim f As SFile = $"{SettingsFolderName}\OFScraperConfigPattern.json"
+                Dim f As SFile = OFScraperConfigPatternFile
                 Dim configText$
-                If Not f.Exists Then
+                CheckOFSConfig()
-                    configText = Text.Encoding.UTF8.GetString(My.Resources.OFResources.OFScraperConfigPattern)
-                    TextSaver.SaveTextToFile(configText, f, True)
-                End If
                 If f.Exists Then
                     Dim replaceValue$ = String.Empty
                     Dim rp As RParams = RParams.DMS(String.Empty, 1, RegexReturn.Replace, RegexOptions.IgnoreCase,
@@ -684,6 +694,7 @@ Namespace API.OnlyFans
                         End If
                         If Settings.FfmpegFile.Exists Then updateConf("ffmpeg", Settings.FfmpegFile.File.ToString.Replace("\", "/"))
                         updateConf("key-mode-default", CStr(MySettings.KeyModeDefault.Value).IfNullOrEmpty(SiteSettings.KeyModeDefault_Default))
+                        updateConf("keydb_api", CStr(MySettings.Keydb_Api.Value))
                         f = currentCache
                         f.Name = "config"
                         f.Extension = "json"

SCrawler/Download/Feed/DownloadFeedForm.vb

@@ -913,21 +913,10 @@ Namespace DownloadObjects
             Try
                 Dim f As SFile = Nothing
                 SessionChooser(False,,,, True, f)
-                If f.Exists Then
+                If Not f.IsEmptyString AndAlso f.Exists Then
-                    Using x As New XmlFile(f, Protector.Modes.All, False) With {.AllowSameNames = True, .XmlReadOnly = True}
+                    Downloader.FilesLoadLastSession(f)
-                        x.LoadData()
+                    FeedChangeMode(FeedModes.Current)
-                        If x.Count > 0 Then
+                    RefillList(True, False)
-                            With Downloader
-                                .Files.Clear()
-                                .Files.ListAddList(x, LAP.NotContainsOnly, LAP.IgnoreICopier)
-                                .FilesLoadLastSession(f)
-                            End With
-                            FeedChangeMode(FeedModes.Current)
-                            RefillList(True, False)
-                        Else
-                            MsgBoxE({"There is no data in the selected session", "Replace current session"}, vbCritical)
-                        End If
-                    End Using
                 End If
             Catch ex As Exception
                 ErrorsDescriber.Execute(EDP.LogMessageValue, ex, "Replace current session")

SCrawler/Download/TDownloader.vb

@@ -177,7 +177,7 @@ Namespace DownloadObjects
                                 Using x As New XmlFile(files.Last, Protector.Modes.All, False) With {.AllowSameNames = True, .XmlReadOnly = True}
                                     x.LoadData()
                                     If x.Count > 0 Then __files.ListAddList(x, LAP.IgnoreICopier)
-                                    If __files.Count > 0 AndAlso (settingValue = 0 OrElse
+                                    If __files.Count > 0 AndAlso (settingValue = 0 OrElse ssfExists OrElse
                                                                   (startTime - {lastDate, __files.Max(Function(f) f.Date)}.Max).TotalMinutes <= settingValue) Then
                                         _Session = __files.Max(Function(f) f.Session)
                                         Me.Files.AddRange(__files)

SCrawler/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.6.4.0")>
+<Assembly: AssemblyVersion("2024.6.6.0")>
-<Assembly: AssemblyFileVersion("2024.6.4.0")>
+<Assembly: AssemblyFileVersion("2024.6.6.0")>
 <Assembly: NeutralResourcesLanguage("en")>

Tools/DeleteGDLTempFiles.bat

@@ -0,0 +1,24 @@
+REM https://superuser.com/a/577640/1410018
+
+@echo off 
+
+set dirname=_MEI
+set usewildcard=true
+set found=false
+if %usewildcard% == true (
+	set dirname=*%dirname%*
+) 
+set directorytosearch=%UserProfile%\AppData\Local\Temp
+echo Searching for %dirname% in %directorytosearch%
+
+for /d %%i in (%directorytosearch%\%dirname%) do (
+	IF EXIST %%i (
+		set found=true
+		echo Deleting the folder %%i
+		rmdir /s /q "%%i"
+	)
+)
+
+if NOT "%found%" == "true" (
+	echo No directories were found with the name of %dirname%
+)