diff --git a/Changelog.md b/Changelog.md index 92b66d8..ba1b3dc 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,19 @@ +# 2024.6.6.0 + +*2024-06-06* + +**ATTENTION!** +1. **To support downloading of DRM protected videos (OnlyFans), please update OF-Scraper to version [3.10](https://github.com/datawhores/OF-Scraper/releases/tag/3.10) (download `zip`, not `exe`).** +2. **If there is a `OFScraperConfigPattern.json` file in the SCrawler settings folder, replace the text of the file with [this text](https://github.com/AAndyProgram/SCrawler/blob/main/SCrawler/API/OnlyFans/OFScraperConfigPattern.json).** +3. **Set the value to `Dynamic rules` (in the site settings) = `https://raw.githubusercontent.com/Growik/onlyfans-dynamic-rules/main/rules.json`.** + +- Added + - OnlyFans: new OF-Scraper option (`keydb_api`) + - Minor improvements +- Fixed + - OnlyFans: **data is not downloading** + - Minor bugs + # 2024.6.4.0 *2024-06-04* diff --git a/ProgramScreenshots/SettingsSiteOnlyFans.png b/ProgramScreenshots/SettingsSiteOnlyFans.png index f4186d7..7aa9a9c 100644 Binary files a/ProgramScreenshots/SettingsSiteOnlyFans.png and b/ProgramScreenshots/SettingsSiteOnlyFans.png differ diff --git a/SCrawler.YouTube/Controls/ButtonRC.vb b/SCrawler.YouTube/Controls/ButtonRC.vb new file mode 100644 index 0000000..b307230 --- /dev/null +++ b/SCrawler.YouTube/Controls/ButtonRC.vb @@ -0,0 +1,23 @@ +' Copyright (C) Andy https://github.com/AAndyProgram +' This program is free software: you can redistribute it and/or modify +' it under the terms of the GNU General Public License as published by +' the Free Software Foundation, either version 3 of the License, or +' (at your option) any later version. +' +' This program is distributed in the hope that it will be useful, +' but WITHOUT ANY WARRANTY +Namespace API.YouTube.Controls + Public Class ButtonRC : Inherits Button + Private Const WM_CONTEXTMENU As Integer = 123 '&H7B + Private Const WM_CANCELMODE As Integer = 31 '&H1F + Private Const WM_INITMENUPOPUP As Integer = 279 '&H117 + Private Const SMTO_NOTIMEOUTIFNOTHUNG As Integer = 8 + Protected Overrides Sub WndProc(ByRef m As Message) + If m.Msg = WM_CONTEXTMENU Or m.Msg = WM_CANCELMODE Or m.Msg = WM_INITMENUPOPUP Or m.Msg = SMTO_NOTIMEOUTIFNOTHUNG Then + m.Result = IntPtr.Zero + Else + MyBase.WndProc(m) + End If + End Sub + End Class +End Namespace \ No newline at end of file diff --git a/SCrawler.YouTube/Controls/VideoOptionsForm.Designer.vb b/SCrawler.YouTube/Controls/VideoOptionsForm.Designer.vb index 56673c8..569e768 100644 --- a/SCrawler.YouTube/Controls/VideoOptionsForm.Designer.vb +++ b/SCrawler.YouTube/Controls/VideoOptionsForm.Designer.vb @@ -65,11 +65,11 @@ Namespace API.YouTube.Controls Me.LBL_TIME = New System.Windows.Forms.Label() Me.LBL_URL = New System.Windows.Forms.LinkLabel() Me.TXT_FILE = New PersonalUtilities.Forms.Controls.ComboBoxExtended() - Me.BTT_BROWSE = New System.Windows.Forms.Button() + Me.BTT_BROWSE = New SCrawler.API.YouTube.Controls.ButtonRC() Me.BTT_DOWN = New System.Windows.Forms.Button() Me.BTT_CANCEL = New System.Windows.Forms.Button() Me.CMB_PLS = New PersonalUtilities.Forms.Controls.ComboBoxExtended() - Me.BTT_PLS_BROWSE = New System.Windows.Forms.Button() + Me.BTT_PLS_BROWSE = New SCrawler.API.YouTube.Controls.ButtonRC() Me.OPT_VIDEO = New System.Windows.Forms.RadioButton() Me.OPT_AUDIO = New System.Windows.Forms.RadioButton() Me.LBL_AUDIO_CODEC = New System.Windows.Forms.Label() @@ -912,13 +912,13 @@ Namespace API.YouTube.Controls Private WithEvents TXT_SUBS_ADDIT As PersonalUtilities.Forms.Controls.TextBoxExtended Private WithEvents TXT_EXTRA_AUDIO_FORMATS As PersonalUtilities.Forms.Controls.TextBoxExtended Private WithEvents TXT_FILE As PersonalUtilities.Forms.Controls.ComboBoxExtended - Private WithEvents BTT_BROWSE As Button + Private WithEvents BTT_BROWSE As SCrawler.API.YouTube.Controls.ButtonRC Private WithEvents BTT_DOWN As Button Private WithEvents BTT_CANCEL As Button Private WithEvents TP_HEADER_INFO_2 As TableLayoutPanel Private WithEvents TXT_FPS As PersonalUtilities.Forms.Controls.TextBoxExtended Private WithEvents CMB_PLS As PersonalUtilities.Forms.Controls.ComboBoxExtended - Private WithEvents BTT_PLS_BROWSE As Button + Private WithEvents BTT_PLS_BROWSE As SCrawler.API.YouTube.Controls.ButtonRC Private WithEvents TXT_AUDIO_BITRATE As PersonalUtilities.Forms.Controls.TextBoxExtended End Class End Namespace \ No newline at end of file diff --git a/SCrawler.YouTube/Controls/VideoOptionsForm.vb b/SCrawler.YouTube/Controls/VideoOptionsForm.vb index a24283f..605a482 100644 --- a/SCrawler.YouTube/Controls/VideoOptionsForm.vb +++ b/SCrawler.YouTube/Controls/VideoOptionsForm.vb @@ -610,7 +610,7 @@ Namespace API.YouTube.Controls $"Video|{AvailableVideoFormats.Select(Function(vf) $"*.{vf.ToLower}").ListToString(";")}" & $"|Audio|{AvailableAudioFormats.Select(Function(af) $"*.{af.ToLower}").ListToString(";")}" f = SFile.SaveAs(f, "Select the destination of the video file",, ext, sPattern, EDP.ReturnValue) - f.Extension = ext + If Not f.IsEmptyString Then f.Extension = ext End If #Enable Warning f = CleanFileName(f) diff --git a/SCrawler.YouTube/My Project/AssemblyInfo.vb b/SCrawler.YouTube/My Project/AssemblyInfo.vb index 9c8577a..f936c5c 100644 --- a/SCrawler.YouTube/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTube/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler.YouTube/SCrawler.YouTube.vbproj b/SCrawler.YouTube/SCrawler.YouTube.vbproj index 8d6d304..75d3904 100644 --- a/SCrawler.YouTube/SCrawler.YouTube.vbproj +++ b/SCrawler.YouTube/SCrawler.YouTube.vbproj @@ -115,6 +115,9 @@ + + Component + ChannelTabsChooserForm.vb diff --git a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb index 8940716..867e77c 100644 --- a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/API/OnlyFans/Declarations.vb b/SCrawler/API/OnlyFans/Declarations.vb index be70fd4..b6d6532 100644 --- a/SCrawler/API/OnlyFans/Declarations.vb +++ b/SCrawler/API/OnlyFans/Declarations.vb @@ -11,5 +11,15 @@ Namespace API.OnlyFans Friend Module Declarations Friend ReadOnly DateProvider As New ADateTime("O") Friend ReadOnly RegExPostID As RParams = RParams.DM("(?<=onlyfans\.com/)(\d+)", 0, EDP.ReturnValue) + Friend ReadOnly OFScraperConfigPatternFile As SFile = $"{SettingsFolderName}\OFScraperConfigPattern.json" + Friend Function CheckOFSConfig() As Boolean + If Not OFScraperConfigPatternFile.Exists Then + Dim t$ = Text.Encoding.UTF8.GetString(My.Resources.OFResources.OFScraperConfigPattern) + TextSaver.SaveTextToFile(t, OFScraperConfigPatternFile, True) + Return OFScraperConfigPatternFile.Exists + Else + Return True + End If + End Function End Module End Namespace \ No newline at end of file diff --git a/SCrawler/API/OnlyFans/OFScraperConfigPattern.json b/SCrawler/API/OnlyFans/OFScraperConfigPattern.json index 98780e4..fb702f7 100644 --- a/SCrawler/API/OnlyFans/OFScraperConfigPattern.json +++ b/SCrawler/API/OnlyFans/OFScraperConfigPattern.json @@ -38,14 +38,18 @@ }, "advanced_options": { "code-execution": false, - "dynamic-mode-default": "deviint", + "dynamic-mode-default": "sneaky", "backend": "aio", "downloadbars": false, "cache-mode": "sqlite", "appendlog": true, "custom": null, "sanitize_text": false, - "avatar": true + "avatar": true, + "custom_values": { + "SNEAKY": "https://raw.githubusercontent.com/Growik/onlyfans-dynamic-rules/main/rules.json", + "CDRM": "https://old.cdrm-project.com/wv" + } }, "responsetype": { "timeline": "Posts", diff --git a/SCrawler/API/OnlyFans/SiteSettings.vb b/SCrawler/API/OnlyFans/SiteSettings.vb index 0922b9c..a3e49fe 100644 --- a/SCrawler/API/OnlyFans/SiteSettings.vb +++ b/SCrawler/API/OnlyFans/SiteSettings.vb @@ -133,12 +133,25 @@ Namespace API.OnlyFans End If End Get End Property + Private ReadOnly Property Keydb_Api_XML As PropertyValue + + Friend ReadOnly Property Keydb_Api As PropertyValue + Get + If Not DefaultInstance Is Nothing Then + Return DirectCast(DefaultInstance, SiteSettings).Keydb_Api_XML + Else + Return Keydb_Api_XML + End If + End Get + End Property #End Region #End Region #Region "Initializer" Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean) MyBase.New("OnlyFans", ".onlyfans.com", AccName, Temp, My.Resources.SiteResources.OnlyFansIcon_32, My.Resources.SiteResources.OnlyFansPic_32) + CheckOFSConfig() + _AllowUserAgentUpdate = False With Responser @@ -192,6 +205,7 @@ Namespace API.OnlyFans End If OFScraperMP4decrypt_XML = New PropertyValue(String.Empty, GetType(String)) KeyModeDefault_XML = New PropertyValue(KeyModeDefault_Default) + Keydb_Api_XML = New PropertyValue(String.Empty, GetType(String)) UserRegex = RParams.DMS(String.Format(UserRegexDefaultPattern, "onlyfans.com/"), 1, EDP.ReturnValue) UrlPatternUser = "https://onlyfans.com/{0}" @@ -204,8 +218,19 @@ Namespace API.OnlyFans End Function #End Region #Region "Update" + Private __UseOldAuthRules As Boolean = True + Private __DynamicRules As String = String.Empty + Friend Overrides Sub BeginUpdate() + __UseOldAuthRules = UseOldAuthRules.Value + __DynamicRules = AConvert(Of String)(DynamicRules.Value, String.Empty) + MyBase.BeginUpdate() + End Sub Friend Overrides Sub Update() - If _SiteEditorFormOpened Then Responser.Cookies.Changed = False + If _SiteEditorFormOpened Then + If Not __UseOldAuthRules = CBool(UseOldAuthRules.Value) Or Not AEquals(Of String)(__DynamicRules, DynamicRules.Value) Then _ + LastDateUpdated = LastDateUpdated.AddYears(-1) + Responser.Cookies.Changed = False + End If MyBase.Update() End Sub #End Region diff --git a/SCrawler/API/OnlyFans/UserData.vb b/SCrawler/API/OnlyFans/UserData.vb index 61f0602..3dff87a 100644 --- a/SCrawler/API/OnlyFans/UserData.vb +++ b/SCrawler/API/OnlyFans/UserData.vb @@ -549,7 +549,8 @@ Namespace API.OnlyFans Optional ByVal Round As Integer = 0) As Boolean Try If UpdateAuthFile(ForceUpdateAuth) Then - Const nullMsg$ = "The auth parameter is null" + Const nullMsg$ = "The auth parameter(s) is null" + Const formatMidPart$ = ":{0}:{1:x}:" Dim j As EContainer Try j = JsonDocument.Parse(AuthFile.GetText) @@ -565,8 +566,16 @@ Namespace API.OnlyFans End Try If Not j Is Nothing Then Dim pattern$ = j.Value("format") - If pattern.IsEmptyString Then Throw New ArgumentNullException("format", nullMsg) - pattern = pattern.Replace("{}", "{0}").Replace("{:x}", "{1:x}") + + If Not pattern.IsEmptyString Then + pattern = pattern.Replace("{}", "{0}").Replace("{:x}", "{1:x}") + ElseIf Not j.Value("prefix").IsEmptyString And Not j.Value("suffix").IsEmptyString Then + pattern = j.Value("prefix") & formatMidPart & j.Value("suffix") + ElseIf Not j.Value("start").IsEmptyString And Not j.Value("end").IsEmptyString Then + pattern = j.Value("start") & formatMidPart & j.Value("end") + Else + Throw New ArgumentNullException("format", nullMsg) + End If Dim li%() = j("checksum_indexes").Select(Function(e) CInt(e(0).Value)).ToArray @@ -607,10 +616,14 @@ Namespace API.OnlyFans Dim r$ = GetWebString(If(ACheck(Of String)(MySettings.DynamicRules.Value), CStr(MySettings.DynamicRules.Value), IIf(MySettings.UseOldAuthRules.Value, urlOld, urlNew)),, EDP.ReturnValue) + Dim checkFormat As Func(Of EContainer, Boolean) = + Function(jj) Not jj.Value("format").IsEmptyString OrElse + (Not jj.Value("prefix").IsEmptyString And Not jj.Value("suffix").IsEmptyString) OrElse + (Not jj.Value("start").IsEmptyString And Not jj.Value("start").IsEmptyString) If Not r.IsEmptyString Then Using j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue) If j.ListExists Then - If Not j.Value("format").IsEmptyString And j("checksum_indexes").ListExists And + If checkFormat(j) And j("checksum_indexes").ListExists And Not j.Value("static_param").IsEmptyString And Not j.Value("checksum_constant").IsEmptyString Then _ TextSaver.SaveTextToFile(r, AuthFile, True, False, EDP.ThrowException) : MySettings.LastDateUpdated = Now End If @@ -658,12 +671,9 @@ Namespace API.OnlyFans currentCache.Validate() Dim cacheRoot As SFile = currentCache.NewPath cacheRoot.Exists(SFO.Path, True, EDP.ThrowException) - Dim f As SFile = $"{SettingsFolderName}\OFScraperConfigPattern.json" + Dim f As SFile = OFScraperConfigPatternFile Dim configText$ - If Not f.Exists Then - configText = Text.Encoding.UTF8.GetString(My.Resources.OFResources.OFScraperConfigPattern) - TextSaver.SaveTextToFile(configText, f, True) - End If + CheckOFSConfig() If f.Exists Then Dim replaceValue$ = String.Empty Dim rp As RParams = RParams.DMS(String.Empty, 1, RegexReturn.Replace, RegexOptions.IgnoreCase, @@ -684,6 +694,7 @@ Namespace API.OnlyFans End If If Settings.FfmpegFile.Exists Then updateConf("ffmpeg", Settings.FfmpegFile.File.ToString.Replace("\", "/")) updateConf("key-mode-default", CStr(MySettings.KeyModeDefault.Value).IfNullOrEmpty(SiteSettings.KeyModeDefault_Default)) + updateConf("keydb_api", CStr(MySettings.Keydb_Api.Value)) f = currentCache f.Name = "config" f.Extension = "json" diff --git a/SCrawler/Download/Feed/DownloadFeedForm.vb b/SCrawler/Download/Feed/DownloadFeedForm.vb index 22ce038..7c0fff1 100644 --- a/SCrawler/Download/Feed/DownloadFeedForm.vb +++ b/SCrawler/Download/Feed/DownloadFeedForm.vb @@ -913,21 +913,10 @@ Namespace DownloadObjects Try Dim f As SFile = Nothing SessionChooser(False,,,, True, f) - If f.Exists Then - Using x As New XmlFile(f, Protector.Modes.All, False) With {.AllowSameNames = True, .XmlReadOnly = True} - x.LoadData() - If x.Count > 0 Then - With Downloader - .Files.Clear() - .Files.ListAddList(x, LAP.NotContainsOnly, LAP.IgnoreICopier) - .FilesLoadLastSession(f) - End With - FeedChangeMode(FeedModes.Current) - RefillList(True, False) - Else - MsgBoxE({"There is no data in the selected session", "Replace current session"}, vbCritical) - End If - End Using + If Not f.IsEmptyString AndAlso f.Exists Then + Downloader.FilesLoadLastSession(f) + FeedChangeMode(FeedModes.Current) + RefillList(True, False) End If Catch ex As Exception ErrorsDescriber.Execute(EDP.LogMessageValue, ex, "Replace current session") diff --git a/SCrawler/Download/TDownloader.vb b/SCrawler/Download/TDownloader.vb index c4a5802..c89c07c 100644 --- a/SCrawler/Download/TDownloader.vb +++ b/SCrawler/Download/TDownloader.vb @@ -177,7 +177,7 @@ Namespace DownloadObjects Using x As New XmlFile(files.Last, Protector.Modes.All, False) With {.AllowSameNames = True, .XmlReadOnly = True} x.LoadData() If x.Count > 0 Then __files.ListAddList(x, LAP.IgnoreICopier) - If __files.Count > 0 AndAlso (settingValue = 0 OrElse + If __files.Count > 0 AndAlso (settingValue = 0 OrElse ssfExists OrElse (startTime - {lastDate, __files.Max(Function(f) f.Date)}.Max).TotalMinutes <= settingValue) Then _Session = __files.Max(Function(f) f.Session) Me.Files.AddRange(__files) diff --git a/SCrawler/My Project/AssemblyInfo.vb b/SCrawler/My Project/AssemblyInfo.vb index 95ece00..50ab392 100644 --- a/SCrawler/My Project/AssemblyInfo.vb +++ b/SCrawler/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/Tools/DeleteGDLTempFiles.bat b/Tools/DeleteGDLTempFiles.bat new file mode 100644 index 0000000..e7d6048 --- /dev/null +++ b/Tools/DeleteGDLTempFiles.bat @@ -0,0 +1,24 @@ +REM https://superuser.com/a/577640/1410018 + +@echo off + +set dirname=_MEI +set usewildcard=true +set found=false +if %usewildcard% == true ( + set dirname=*%dirname%* +) +set directorytosearch=%UserProfile%\AppData\Local\Temp +echo Searching for %dirname% in %directorytosearch% + +for /d %%i in (%directorytosearch%\%dirname%) do ( + IF EXIST %%i ( + set found=true + echo Deleting the folder %%i + rmdir /s /q "%%i" + ) +) + +if NOT "%found%" == "true" ( + echo No directories were found with the name of %dirname% +) \ No newline at end of file