2024.6.6.0

YT VideoOptionsForm: file path is cleared when the cancel button is clicked (browse button); remove the context menu when the right clicking on browse button; add 'ButtonRC' class SCrawler API.OnlyFans: add check config to the SiteSettings; update config; add 'Keydb_Api' property; reset 'LastDateUpdated' when rules change; add support 'prefix/suffix' and 'start/end' to support other rules formats DownloadFeedForm: update 'BTT_CURR_SESSION_SET_Click' function TDownloader: update 'FilesLoadLastSession' function
2026-03-14 15:52:18 +00:00 · 2024-06-06 05:49:50 +03:00
parent 2ae8c3acfc
commit 93ea2a55ac
16 changed files with 144 additions and 39 deletions
--- a/Changelog.md
+++ b/Changelog.md
@@ -1,3 +1,19 @@
+# 2024.6.6.0
+
+*2024-06-06*
+
+**ATTENTION!**
+1. **To support downloading of DRM protected videos (OnlyFans), please update OF-Scraper to version [3.10](https://github.com/datawhores/OF-Scraper/releases/tag/3.10) (download `zip`, not `exe`).**
+2. **If there is a `OFScraperConfigPattern.json` file in the SCrawler settings folder, replace the text of the file with [this text](https://github.com/AAndyProgram/SCrawler/blob/main/SCrawler/API/OnlyFans/OFScraperConfigPattern.json).**
+3. **Set the value to `Dynamic rules` (in the site settings) = `https://raw.githubusercontent.com/Growik/onlyfans-dynamic-rules/main/rules.json`.**
+
+- Added
+  - OnlyFans: new OF-Scraper option (`keydb_api`)
+  - Minor improvements
+- Fixed
+  - OnlyFans: **data is not downloading**
+  - Minor bugs
+
 # 2024.6.4.0

 *2024-06-04*
--- a/ProgramScreenshots/SettingsSiteOnlyFans.png
+++ b/ProgramScreenshots/SettingsSiteOnlyFans.png
--- a/SCrawler.YouTube/Controls/ButtonRC.vb
+++ b/SCrawler.YouTube/Controls/ButtonRC.vb
@@ -0,0 +1,23 @@
+' Copyright (C) Andy https://github.com/AAndyProgram
+' This program is free software: you can redistribute it and/or modify
+' it under the terms of the GNU General Public License as published by
+' the Free Software Foundation, either version 3 of the License, or
+' (at your option) any later version.
+'
+' This program is distributed in the hope that it will be useful,
+' but WITHOUT ANY WARRANTY
+Namespace API.YouTube.Controls
+    Public Class ButtonRC : Inherits Button
+        Private Const WM_CONTEXTMENU As Integer = 123 '&H7B
+        Private Const WM_CANCELMODE As Integer = 31 '&H1F
+        Private Const WM_INITMENUPOPUP As Integer = 279 '&H117
+        Private Const SMTO_NOTIMEOUTIFNOTHUNG As Integer = 8
+        Protected Overrides Sub WndProc(ByRef m As Message)
+            If m.Msg = WM_CONTEXTMENU Or m.Msg = WM_CANCELMODE Or m.Msg = WM_INITMENUPOPUP Or m.Msg = SMTO_NOTIMEOUTIFNOTHUNG Then
+                m.Result = IntPtr.Zero
+            Else
+                MyBase.WndProc(m)
+            End If
+        End Sub
+    End Class
+End Namespace
--- a/SCrawler.YouTube/Controls/VideoOptionsForm.Designer.vb
+++ b/SCrawler.YouTube/Controls/VideoOptionsForm.Designer.vb
@@ -65,11 +65,11 @@ Namespace API.YouTube.Controls
            Me.LBL_TIME = New System.Windows.Forms.Label()
            Me.LBL_URL = New System.Windows.Forms.LinkLabel()
            Me.TXT_FILE = New PersonalUtilities.Forms.Controls.ComboBoxExtended()
-            Me.BTT_BROWSE = New System.Windows.Forms.Button()
+            Me.BTT_BROWSE = New SCrawler.API.YouTube.Controls.ButtonRC()
            Me.BTT_DOWN = New System.Windows.Forms.Button()
            Me.BTT_CANCEL = New System.Windows.Forms.Button()
            Me.CMB_PLS = New PersonalUtilities.Forms.Controls.ComboBoxExtended()
-            Me.BTT_PLS_BROWSE = New System.Windows.Forms.Button()
+            Me.BTT_PLS_BROWSE = New SCrawler.API.YouTube.Controls.ButtonRC()
            Me.OPT_VIDEO = New System.Windows.Forms.RadioButton()
            Me.OPT_AUDIO = New System.Windows.Forms.RadioButton()
            Me.LBL_AUDIO_CODEC = New System.Windows.Forms.Label()
@@ -912,13 +912,13 @@ Namespace API.YouTube.Controls
        Private WithEvents TXT_SUBS_ADDIT As PersonalUtilities.Forms.Controls.TextBoxExtended
        Private WithEvents TXT_EXTRA_AUDIO_FORMATS As PersonalUtilities.Forms.Controls.TextBoxExtended
        Private WithEvents TXT_FILE As PersonalUtilities.Forms.Controls.ComboBoxExtended
-        Private WithEvents BTT_BROWSE As Button
+        Private WithEvents BTT_BROWSE As SCrawler.API.YouTube.Controls.ButtonRC
        Private WithEvents BTT_DOWN As Button
        Private WithEvents BTT_CANCEL As Button
        Private WithEvents TP_HEADER_INFO_2 As TableLayoutPanel
        Private WithEvents TXT_FPS As PersonalUtilities.Forms.Controls.TextBoxExtended
        Private WithEvents CMB_PLS As PersonalUtilities.Forms.Controls.ComboBoxExtended
-        Private WithEvents BTT_PLS_BROWSE As Button
+        Private WithEvents BTT_PLS_BROWSE As SCrawler.API.YouTube.Controls.ButtonRC
        Private WithEvents TXT_AUDIO_BITRATE As PersonalUtilities.Forms.Controls.TextBoxExtended
    End Class
 End Namespace
--- a/SCrawler.YouTube/Controls/VideoOptionsForm.vb
+++ b/SCrawler.YouTube/Controls/VideoOptionsForm.vb
@@ -610,7 +610,7 @@ Namespace API.YouTube.Controls
                                $"Video|{AvailableVideoFormats.Select(Function(vf) $"*.{vf.ToLower}").ListToString(";")}" &
                                $"|Audio|{AvailableAudioFormats.Select(Function(af) $"*.{af.ToLower}").ListToString(";")}"
                f = SFile.SaveAs(f, "Select the destination of the video file",, ext, sPattern, EDP.ReturnValue)
-                f.Extension = ext
+                If Not f.IsEmptyString Then f.Extension = ext
            End If
 #Enable Warning
            f = CleanFileName(f)
--- a/Project/AssemblyInfo.vb
+++ b/Project/AssemblyInfo.vb
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>

-<Assembly: AssemblyVersion("2024.6.4.0")>
-<Assembly: AssemblyFileVersion("2024.6.4.0")>
+<Assembly: AssemblyVersion("2024.6.6.0")>
+<Assembly: AssemblyFileVersion("2024.6.6.0")>
 <Assembly: NeutralResourcesLanguage("en")>
--- a/SCrawler.YouTube/SCrawler.YouTube.vbproj
+++ b/SCrawler.YouTube/SCrawler.YouTube.vbproj
@@ -115,6 +115,9 @@
  <ItemGroup>
    <Compile Include="Attributes\GridVisibleAttribute.vb" />
    <Compile Include="Base\TableControlsProcessor.vb" />
+    <Compile Include="Controls\ButtonRC.vb">
+      <SubType>Component</SubType>
+    </Compile>
    <Compile Include="Controls\ChannelTabsChooserForm.Designer.vb">
      <DependentUpon>ChannelTabsChooserForm.vb</DependentUpon>
    </Compile>
--- a/SCrawler.YouTubeDownloader/My
+++ b/SCrawler.YouTubeDownloader/My
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>

-<Assembly: AssemblyVersion("2024.6.4.0")>
-<Assembly: AssemblyFileVersion("2024.6.4.0")>
+<Assembly: AssemblyVersion("2024.6.6.0")>
+<Assembly: AssemblyFileVersion("2024.6.6.0")>
 <Assembly: NeutralResourcesLanguage("en")>
--- a/SCrawler/API/OnlyFans/Declarations.vb
+++ b/SCrawler/API/OnlyFans/Declarations.vb
@@ -11,5 +11,15 @@ Namespace API.OnlyFans
    Friend Module Declarations
        Friend ReadOnly DateProvider As New ADateTime("O")
        Friend ReadOnly RegExPostID As RParams = RParams.DM("(?<=onlyfans\.com/)(\d+)", 0, EDP.ReturnValue)
+        Friend ReadOnly OFScraperConfigPatternFile As SFile = $"{SettingsFolderName}\OFScraperConfigPattern.json"
+        Friend Function CheckOFSConfig() As Boolean
+            If Not OFScraperConfigPatternFile.Exists Then
+                Dim t$ = Text.Encoding.UTF8.GetString(My.Resources.OFResources.OFScraperConfigPattern)
+                TextSaver.SaveTextToFile(t, OFScraperConfigPatternFile, True)
+                Return OFScraperConfigPatternFile.Exists
+            Else
+                Return True
+            End If
+        End Function
    End Module
 End Namespace
--- a/SCrawler/API/OnlyFans/OFScraperConfigPattern.json
+++ b/SCrawler/API/OnlyFans/OFScraperConfigPattern.json
@@ -38,14 +38,18 @@
        },
        "advanced_options": {
            "code-execution": false,
-            "dynamic-mode-default": "deviint",
+            "dynamic-mode-default": "sneaky",
            "backend": "aio",
            "downloadbars": false,
            "cache-mode": "sqlite",
            "appendlog": true,
            "custom": null,
            "sanitize_text": false,
-            "avatar": true
+            "avatar": true,
+            "custom_values": {
+                "SNEAKY": "https://raw.githubusercontent.com/Growik/onlyfans-dynamic-rules/main/rules.json",
+                "CDRM": "https://old.cdrm-project.com/wv"
+            }
        },
        "responsetype": {
            "timeline": "Posts",
--- a/SCrawler/API/OnlyFans/SiteSettings.vb
+++ b/SCrawler/API/OnlyFans/SiteSettings.vb
@@ -133,12 +133,25 @@ Namespace API.OnlyFans
                End If
            End Get
        End Property
+        <PClonable, PXML("keydb_api")> Private ReadOnly Property Keydb_Api_XML As PropertyValue
+        <PropertyOption(ControlText:="keydb_api", Category:=CAT_OFS)>
+        Friend ReadOnly Property Keydb_Api As PropertyValue
+            Get
+                If Not DefaultInstance Is Nothing Then
+                    Return DirectCast(DefaultInstance, SiteSettings).Keydb_Api_XML
+                Else
+                    Return Keydb_Api_XML
+                End If
+            End Get
+        End Property
 #End Region
 #End Region
 #Region "Initializer"
        Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean)
            MyBase.New("OnlyFans", ".onlyfans.com", AccName, Temp, My.Resources.SiteResources.OnlyFansIcon_32, My.Resources.SiteResources.OnlyFansPic_32)

+            CheckOFSConfig()
+
            _AllowUserAgentUpdate = False

            With Responser
@@ -192,6 +205,7 @@ Namespace API.OnlyFans
            End If
            OFScraperMP4decrypt_XML = New PropertyValue(String.Empty, GetType(String))
            KeyModeDefault_XML = New PropertyValue(KeyModeDefault_Default)
+            Keydb_Api_XML = New PropertyValue(String.Empty, GetType(String))

            UserRegex = RParams.DMS(String.Format(UserRegexDefaultPattern, "onlyfans.com/"), 1, EDP.ReturnValue)
            UrlPatternUser = "https://onlyfans.com/{0}"
@@ -204,8 +218,19 @@ Namespace API.OnlyFans
        End Function
 #End Region
 #Region "Update"
+        Private __UseOldAuthRules As Boolean = True
+        Private __DynamicRules As String = String.Empty
+        Friend Overrides Sub BeginUpdate()
+            __UseOldAuthRules = UseOldAuthRules.Value
+            __DynamicRules = AConvert(Of String)(DynamicRules.Value, String.Empty)
+            MyBase.BeginUpdate()
+        End Sub
        Friend Overrides Sub Update()
-            If _SiteEditorFormOpened Then Responser.Cookies.Changed = False
+            If _SiteEditorFormOpened Then
+                If Not __UseOldAuthRules = CBool(UseOldAuthRules.Value) Or Not AEquals(Of String)(__DynamicRules, DynamicRules.Value) Then _
+                   LastDateUpdated = LastDateUpdated.AddYears(-1)
+                Responser.Cookies.Changed = False
+            End If
            MyBase.Update()
        End Sub
 #End Region
--- a/SCrawler/API/OnlyFans/UserData.vb
+++ b/SCrawler/API/OnlyFans/UserData.vb
@@ -549,7 +549,8 @@ Namespace API.OnlyFans
                                         Optional ByVal Round As Integer = 0) As Boolean
            Try
                If UpdateAuthFile(ForceUpdateAuth) Then
-                    Const nullMsg$ = "The auth parameter is null"
+                    Const nullMsg$ = "The auth parameter(s) is null"
+                    Const formatMidPart$ = ":{0}:{1:x}:"
                    Dim j As EContainer
                    Try
                        j = JsonDocument.Parse(AuthFile.GetText)
@@ -565,8 +566,16 @@ Namespace API.OnlyFans
                    End Try
                    If Not j Is Nothing Then
                        Dim pattern$ = j.Value("format")
-                        If pattern.IsEmptyString Then Throw New ArgumentNullException("format", nullMsg)
-                        pattern = pattern.Replace("{}", "{0}").Replace("{:x}", "{1:x}")
+
+                        If Not pattern.IsEmptyString Then
+                            pattern = pattern.Replace("{}", "{0}").Replace("{:x}", "{1:x}")
+                        ElseIf Not j.Value("prefix").IsEmptyString And Not j.Value("suffix").IsEmptyString Then
+                            pattern = j.Value("prefix") & formatMidPart & j.Value("suffix")
+                        ElseIf Not j.Value("start").IsEmptyString And Not j.Value("end").IsEmptyString Then
+                            pattern = j.Value("start") & formatMidPart & j.Value("end")
+                        Else
+                            Throw New ArgumentNullException("format", nullMsg)
+                        End If

                        Dim li%() = j("checksum_indexes").Select(Function(e) CInt(e(0).Value)).ToArray

@@ -607,10 +616,14 @@ Namespace API.OnlyFans
                    Dim r$ = GetWebString(If(ACheck(Of String)(MySettings.DynamicRules.Value),
                                             CStr(MySettings.DynamicRules.Value),
                                             IIf(MySettings.UseOldAuthRules.Value, urlOld, urlNew)),, EDP.ReturnValue)
+                    Dim checkFormat As Func(Of EContainer, Boolean) =
+                        Function(jj) Not jj.Value("format").IsEmptyString OrElse
+                                     (Not jj.Value("prefix").IsEmptyString And Not jj.Value("suffix").IsEmptyString) OrElse
+                                     (Not jj.Value("start").IsEmptyString And Not jj.Value("start").IsEmptyString)
                    If Not r.IsEmptyString Then
                        Using j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue)
                            If j.ListExists Then
-                                If Not j.Value("format").IsEmptyString And j("checksum_indexes").ListExists And
+                                If checkFormat(j) And j("checksum_indexes").ListExists And
                                   Not j.Value("static_param").IsEmptyString And Not j.Value("checksum_constant").IsEmptyString Then _
                                   TextSaver.SaveTextToFile(r, AuthFile, True, False, EDP.ThrowException) : MySettings.LastDateUpdated = Now
                            End If
@@ -658,12 +671,9 @@ Namespace API.OnlyFans
                currentCache.Validate()
                Dim cacheRoot As SFile = currentCache.NewPath
                cacheRoot.Exists(SFO.Path, True, EDP.ThrowException)
-                Dim f As SFile = $"{SettingsFolderName}\OFScraperConfigPattern.json"
+                Dim f As SFile = OFScraperConfigPatternFile
                Dim configText$
-                If Not f.Exists Then
-                    configText = Text.Encoding.UTF8.GetString(My.Resources.OFResources.OFScraperConfigPattern)
-                    TextSaver.SaveTextToFile(configText, f, True)
-                End If
+                CheckOFSConfig()
                If f.Exists Then
                    Dim replaceValue$ = String.Empty
                    Dim rp As RParams = RParams.DMS(String.Empty, 1, RegexReturn.Replace, RegexOptions.IgnoreCase,
@@ -684,6 +694,7 @@ Namespace API.OnlyFans
                        End If
                        If Settings.FfmpegFile.Exists Then updateConf("ffmpeg", Settings.FfmpegFile.File.ToString.Replace("\", "/"))
                        updateConf("key-mode-default", CStr(MySettings.KeyModeDefault.Value).IfNullOrEmpty(SiteSettings.KeyModeDefault_Default))
+                        updateConf("keydb_api", CStr(MySettings.Keydb_Api.Value))
                        f = currentCache
                        f.Name = "config"
                        f.Extension = "json"
--- a/SCrawler/Download/Feed/DownloadFeedForm.vb
+++ b/SCrawler/Download/Feed/DownloadFeedForm.vb
@@ -913,21 +913,10 @@ Namespace DownloadObjects
            Try
                Dim f As SFile = Nothing
                SessionChooser(False,,,, True, f)
-                If f.Exists Then
-                    Using x As New XmlFile(f, Protector.Modes.All, False) With {.AllowSameNames = True, .XmlReadOnly = True}
-                        x.LoadData()
-                        If x.Count > 0 Then
-                            With Downloader
-                                .Files.Clear()
-                                .Files.ListAddList(x, LAP.NotContainsOnly, LAP.IgnoreICopier)
-                                .FilesLoadLastSession(f)
-                            End With
-                            FeedChangeMode(FeedModes.Current)
-                            RefillList(True, False)
-                        Else
-                            MsgBoxE({"There is no data in the selected session", "Replace current session"}, vbCritical)
-                        End If
-                    End Using
+                If Not f.IsEmptyString AndAlso f.Exists Then
+                    Downloader.FilesLoadLastSession(f)
+                    FeedChangeMode(FeedModes.Current)
+                    RefillList(True, False)
                End If
            Catch ex As Exception
                ErrorsDescriber.Execute(EDP.LogMessageValue, ex, "Replace current session")
--- a/SCrawler/Download/TDownloader.vb
+++ b/SCrawler/Download/TDownloader.vb
@@ -177,7 +177,7 @@ Namespace DownloadObjects
                                Using x As New XmlFile(files.Last, Protector.Modes.All, False) With {.AllowSameNames = True, .XmlReadOnly = True}
                                    x.LoadData()
                                    If x.Count > 0 Then __files.ListAddList(x, LAP.IgnoreICopier)
-                                    If __files.Count > 0 AndAlso (settingValue = 0 OrElse
+                                    If __files.Count > 0 AndAlso (settingValue = 0 OrElse ssfExists OrElse
                                                                  (startTime - {lastDate, __files.Max(Function(f) f.Date)}.Max).TotalMinutes <= settingValue) Then
                                        _Session = __files.Max(Function(f) f.Session)
                                        Me.Files.AddRange(__files)
--- a/Project/AssemblyInfo.vb
+++ b/Project/AssemblyInfo.vb
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>

-<Assembly: AssemblyVersion("2024.6.4.0")>
-<Assembly: AssemblyFileVersion("2024.6.4.0")>
+<Assembly: AssemblyVersion("2024.6.6.0")>
+<Assembly: AssemblyFileVersion("2024.6.6.0")>
 <Assembly: NeutralResourcesLanguage("en")>
--- a/Tools/DeleteGDLTempFiles.bat
+++ b/Tools/DeleteGDLTempFiles.bat
@@ -0,0 +1,24 @@
+REM https://superuser.com/a/577640/1410018
+
+@echo off 
+
+set dirname=_MEI
+set usewildcard=true
+set found=false
+if %usewildcard% == true (
+	set dirname=*%dirname%*
+) 
+set directorytosearch=%UserProfile%\AppData\Local\Temp
+echo Searching for %dirname% in %directorytosearch%
+
+for /d %%i in (%directorytosearch%\%dirname%) do (
+	IF EXIST %%i (
+		set found=true
+		echo Deleting the folder %%i
+		rmdir /s /q "%%i"
+	)
+)
+
+if NOT "%found%" == "true" (
+	echo No directories were found with the name of %dirname%
+)