mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-19 02:02:17 +00:00
2023.6.19.0
YT.Progress: make the playlists parsing progress more informative; change form display method YT.YouTubeMediaContainerBase: fix sort algo YT.Tray: add 'Add' button; add 'Ctrl+Click' on tray icon to add download YT.Settings: add setting 'Download on click in tray: show form' LPSG: some files didn't download (encoding) Twitter: hide cache deletion errors Mastogon: fixed bug in 'ReparseMissing' function Reddit: downloaded gifs are static XHamster: videos are not downloading or downloading incorrectly Progress: fix bugs; minor improvements
This commit is contained in:
Andy
15
SCrawler/API/OnlyFans/Declarations.vb
Normal file
15
SCrawler/API/OnlyFans/Declarations.vb
Normal file
@@ -0,0 +1,15 @@
|
||||
' Copyright (C) 2023 Andy https://github.com/AAndyProgram
|
||||
' This program is free software: you can redistribute it and/or modify
|
||||
' it under the terms of the GNU General Public License as published by
|
||||
' the Free Software Foundation, either version 3 of the License, or
|
||||
' (at your option) any later version.
|
||||
'
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports PersonalUtilities.Functions.RegularExpressions
|
||||
Namespace API.OnlyFans
|
||||
Friend Module Declarations
|
||||
Friend ReadOnly DateProvider As New ADateTime("O")
|
||||
Friend ReadOnly RegExPostID As RParams = RParams.DM("(?<=onlyfans\.com/)(\d+)", 0, EDP.ReturnValue)
|
||||
End Module
|
||||
End Namespace
|
||||
165
SCrawler/API/OnlyFans/SiteSettings.vb
Normal file
165
SCrawler/API/OnlyFans/SiteSettings.vb
Normal file
@@ -0,0 +1,165 @@
|
||||
' Copyright (C) 2023 Andy https://github.com/AAndyProgram
|
||||
' This program is free software: you can redistribute it and/or modify
|
||||
' it under the terms of the GNU General Public License as published by
|
||||
' the Free Software Foundation, either version 3 of the License, or
|
||||
' (at your option) any later version.
|
||||
'
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports SCrawler.API.Base
|
||||
Imports SCrawler.Plugin
|
||||
Imports SCrawler.Plugin.Attributes
|
||||
Imports PersonalUtilities.Forms
|
||||
Imports PersonalUtilities.Tools.Web.Clients
|
||||
Imports PersonalUtilities.Tools.Web.Cookies
|
||||
Imports PersonalUtilities.Functions.RegularExpressions
|
||||
Namespace API.OnlyFans
|
||||
<Manifest("AndyProgram_OnlyFans"), SavedPosts, SeparatedTasks(1)>
|
||||
Friend Class SiteSettings : Inherits SiteSettingsBase
|
||||
#Region "Icon"
|
||||
Friend Overrides ReadOnly Property Icon As Icon
|
||||
Get
|
||||
Return My.Resources.SiteResources.OnlyFansIcon_32
|
||||
End Get
|
||||
End Property
|
||||
Friend Overrides ReadOnly Property Image As Image
|
||||
Get
|
||||
Return My.Resources.SiteResources.OnlyFansPic_32
|
||||
End Get
|
||||
End Property
|
||||
#End Region
|
||||
#Region "Declarations"
|
||||
Private Const HeaderBrowser As String = "sec-ch-ua"
|
||||
Private Const HeaderUserID As String = "User-Id"
|
||||
Private Const HeaderXBC As String = "X-Bc"
|
||||
Private Const HeaderAppToken As String = "App-Token"
|
||||
<PropertyOption(ControlText:=HeaderUserID, AllowNull:=False)>
|
||||
Friend ReadOnly Property HH_USER_ID As PropertyValue
|
||||
<PropertyOption(ControlText:=HeaderXBC, AllowNull:=False)>
|
||||
Private ReadOnly Property HH_X_BC As PropertyValue
|
||||
<PropertyOption(ControlText:=HeaderAppToken, AllowNull:=False)>
|
||||
Private ReadOnly Property HH_APP_TOKEN As PropertyValue
|
||||
<PropertyOption(ControlText:=HeaderBrowser, AllowNull:=False)>
|
||||
Private ReadOnly Property HH_BROWSER As PropertyValue
|
||||
<PropertyOption(AllowNull:=False)>
|
||||
Private ReadOnly Property UserAgent As PropertyValue
|
||||
Private Sub UpdateHeader(ByVal PropertyName As String, ByVal Value As String)
|
||||
Dim hName$ = String.Empty
|
||||
Dim isUserAgent As Boolean = False
|
||||
Select Case PropertyName
|
||||
Case NameOf(HH_USER_ID) : hName = HeaderUserID
|
||||
Case NameOf(HH_X_BC) : hName = HeaderXBC
|
||||
Case NameOf(HH_APP_TOKEN) : hName = HeaderAppToken
|
||||
Case NameOf(HH_BROWSER) : hName = HeaderBrowser
|
||||
Case NameOf(UserAgent) : isUserAgent = True
|
||||
End Select
|
||||
If Not hName.IsEmptyString Then
|
||||
Responser.Headers.Add(hName, Value)
|
||||
ElseIf isUserAgent Then
|
||||
Responser.UserAgent = Value
|
||||
End If
|
||||
End Sub
|
||||
<PXML("LastDateUpdated")> Private ReadOnly Property LastDateUpdated_XML As PropertyValue
|
||||
Friend Property LastDateUpdated As Date
|
||||
Get
|
||||
Return LastDateUpdated_XML.Value
|
||||
End Get
|
||||
Set(ByVal d As Date)
|
||||
LastDateUpdated_XML.Value = d
|
||||
End Set
|
||||
End Property
|
||||
<PropertyOption(ControlText:="Use old authorization rules",
|
||||
ControlToolTip:="Use old dynamic rules (from 'DATAHOARDERS') or new ones (from 'DIGITALCRIMINALS')." & vbCr &
|
||||
"Change this value only if you know what you are doing."), PXML>
|
||||
Friend ReadOnly Property UseOldAuthRules As PropertyValue
|
||||
<PropertyOption(ControlText:="Dynamic rules update", ControlToolTip:="'Dynamic rules' update interval (minutes). Default: 1440", LeftOffset:=110), PXML>
|
||||
Friend ReadOnly Property DynamicRulesUpdateInterval As PropertyValue
|
||||
<Provider(NameOf(DynamicRulesUpdateInterval), FieldsChecker:=True)>
|
||||
Private ReadOnly Property DynamicRulesUpdateIntervalProvider As IFormatProvider
|
||||
<PropertyOption(ControlText:="Dynamic rules",
|
||||
ControlToolTip:="Overwrite 'Dynamic rules' with this URL" & vbCr &
|
||||
"Change this value only if you know what you are doing."), PXML>
|
||||
Friend ReadOnly Property DynamicRules As PropertyValue
|
||||
#End Region
|
||||
#Region "Initializer"
|
||||
Friend Sub New()
|
||||
MyBase.New("OnlyFans", ".onlyfans.com")
|
||||
|
||||
With Responser
|
||||
.Accept = "application/json, text/plain, */*"
|
||||
.AutomaticDecompression = Net.DecompressionMethods.GZip
|
||||
.CookiesExtractMode = Responser.CookiesExtractModes.Any
|
||||
.CookiesExtractedAutoSave = False
|
||||
.CookiesUpdateMode = CookieKeeper.UpdateModes.Disabled
|
||||
.Cookies.ChangedAllowInternalDrop = False
|
||||
.Cookies.Changed = False
|
||||
With .Headers
|
||||
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform))
|
||||
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaMobile))
|
||||
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest))
|
||||
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode))
|
||||
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite))
|
||||
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.DHT))
|
||||
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.Authority, "onlyfans.com"))
|
||||
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.AcceptEncoding))
|
||||
HH_USER_ID = New PropertyValue(.Value(HeaderUserID), GetType(String), Sub(v) UpdateHeader(NameOf(HH_USER_ID), v))
|
||||
HH_X_BC = New PropertyValue(.Value(HeaderXBC), GetType(String), Sub(v) UpdateHeader(NameOf(HH_X_BC), v))
|
||||
HH_APP_TOKEN = New PropertyValue(.Value(HeaderAppToken), GetType(String), Sub(v) UpdateHeader(NameOf(HH_APP_TOKEN), v))
|
||||
HH_BROWSER = New PropertyValue(.Value(HeaderBrowser), GetType(String), Sub(v) UpdateHeader(NameOf(HH_BROWSER), v))
|
||||
End With
|
||||
UserAgent = New PropertyValue(IIf(.UserAgentExists, .UserAgent, String.Empty), GetType(String), Sub(v) UpdateHeader(NameOf(UserAgent), v))
|
||||
End With
|
||||
|
||||
LastDateUpdated_XML = New PropertyValue(Now.AddYears(-1), GetType(Date))
|
||||
UseOldAuthRules = New PropertyValue(False)
|
||||
DynamicRulesUpdateInterval = New PropertyValue(60 * 24)
|
||||
DynamicRulesUpdateIntervalProvider = New FieldsCheckerProviderSimple(Function(v) IIf(AConvert(Of Integer)(v, 0) > 0, v, Nothing),
|
||||
"The value of [{0}] field must be greater than 0")
|
||||
DynamicRules = New PropertyValue(String.Empty, GetType(String))
|
||||
UserRegex = RParams.DMS("onlyfans.com/(\w+)", 1, EDP.ReturnValue)
|
||||
UrlPatternUser = "https://onlyfans.com/{0}"
|
||||
ImageVideoContains = "onlyfans.com"
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "GetInstance"
|
||||
Friend Overrides Function GetInstance(ByVal What As ISiteSettings.Download) As IPluginContentProvider
|
||||
Return New UserData
|
||||
End Function
|
||||
#End Region
|
||||
#Region "Update"
|
||||
Friend Overrides Sub Update()
|
||||
If _SiteEditorFormOpened Then Responser.Cookies.Changed = False
|
||||
MyBase.Update()
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "Download"
|
||||
Friend Overrides Function BaseAuthExists() As Boolean
|
||||
Return Responser.CookiesExists And {HH_USER_ID, HH_X_BC, HH_APP_TOKEN, HH_BROWSER, UserAgent}.All(Function(v) ACheck(v.Value))
|
||||
End Function
|
||||
Friend Overrides Function ReadyToDownload(ByVal What As ISiteSettings.Download) As Boolean
|
||||
Return BaseAuthExists() And Not SessionAborted
|
||||
End Function
|
||||
Friend Property SessionAborted As Boolean = False
|
||||
Friend Overrides Sub AfterDownload(ByVal User As Object, ByVal What As ISiteSettings.Download)
|
||||
Responser.Cookies.Update(DirectCast(User, UserData).CCookie)
|
||||
End Sub
|
||||
Friend Overrides Sub DownloadDone(ByVal What As ISiteSettings.Download)
|
||||
MyBase.DownloadDone(What)
|
||||
SessionAborted = False
|
||||
If Responser.Cookies.Changed Then Responser.SaveCookies() : Responser.Cookies.Changed = False
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "GetUserUrl, GetUserPostUrl"
|
||||
Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String
|
||||
Return String.Format(UrlPatternUser, If(User.ID.IsEmptyString, User.Name, $"u{User.ID}"))
|
||||
End Function
|
||||
Friend Overrides Function GetUserPostUrl(ByVal User As UserDataBase, ByVal Media As UserMedia) As String
|
||||
If Not Media.Post.ID.IsEmptyString Then
|
||||
Return String.Format("https://onlyfans.com/{0}/{1}", Media.Post.ID, If(User.ID.IsEmptyString, User.Name, $"u{User.ID}"))
|
||||
Else
|
||||
Return String.Empty
|
||||
End If
|
||||
End Function
|
||||
#End Region
|
||||
End Class
|
||||
End Namespace
|
||||
362
SCrawler/API/OnlyFans/UserData.vb
Normal file
362
SCrawler/API/OnlyFans/UserData.vb
Normal file
@@ -0,0 +1,362 @@
|
||||
' Copyright (C) 2023 Andy https://github.com/AAndyProgram
|
||||
' This program is free software: you can redistribute it and/or modify
|
||||
' it under the terms of the GNU General Public License as published by
|
||||
' the Free Software Foundation, either version 3 of the License, or
|
||||
' (at your option) any later version.
|
||||
'
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports System.Threading
|
||||
Imports SCrawler.API.Base
|
||||
Imports SCrawler.API.YouTube.Objects
|
||||
Imports PersonalUtilities.Functions.XML
|
||||
Imports PersonalUtilities.Functions.RegularExpressions
|
||||
Imports PersonalUtilities.Tools.Web.Clients
|
||||
Imports PersonalUtilities.Tools.Web.Clients.EventArguments
|
||||
Imports PersonalUtilities.Tools.Web.Cookies
|
||||
Imports PersonalUtilities.Tools.Web.Documents.JSON
|
||||
Imports UTypes = SCrawler.API.Base.UserMedia.Types
|
||||
Imports UStates = SCrawler.API.Base.UserMedia.States
|
||||
Namespace API.OnlyFans
|
||||
Friend Class UserData : Inherits UserDataBase
|
||||
#Region "Declarations"
|
||||
Friend Property CCookie As CookieKeeper = Nothing
|
||||
Private Const HeaderSign As String = "Sign"
|
||||
Private Const HeaderTime As String = "Time"
|
||||
Private ReadOnly Property MySettings As SiteSettings
|
||||
Get
|
||||
Return HOST.Source
|
||||
End Get
|
||||
End Property
|
||||
Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean)
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "Download functions"
|
||||
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
|
||||
If Not CCookie Is Nothing Then CCookie.Dispose()
|
||||
CCookie = Responser.Cookies.Copy
|
||||
Responser.Cookies.Clear()
|
||||
AddHandler Responser.ResponseReceived, AddressOf OnResponseReceived
|
||||
UpdateCookieHeader()
|
||||
DownloadData(IIf(IsSavedPosts, 0, String.Empty), Token)
|
||||
End Sub
|
||||
Private Sub OnResponseReceived(ByVal Sender As Object, ByVal e As WebDataResponse)
|
||||
If e.CookiesExists Then
|
||||
CCookie.Update(e.Cookies, CookieKeeper.UpdateModes.ReplaceByNameAll,, EDP.ReturnValue)
|
||||
UpdateCookieHeader()
|
||||
End If
|
||||
End Sub
|
||||
Private Sub UpdateCookieHeader()
|
||||
Responser.Headers.Add("Cookie", CCookie.ToString(False))
|
||||
End Sub
|
||||
Private Const BaseUrlPattern As String = "https://onlyfans.com{0}"
|
||||
Private Overloads Sub DownloadData(ByVal Cursor As String, ByVal Token As CancellationToken)
|
||||
|
||||
Dim url$ = String.Empty
|
||||
Dim _complete As Boolean = True
|
||||
Do
|
||||
Try
|
||||
Dim tmpCursor$ = String.Empty
|
||||
Dim hasMore As Boolean = False
|
||||
Dim path$ = String.Empty
|
||||
Dim postDate$, postID$
|
||||
Dim n As EContainer
|
||||
Dim mediaList As List(Of UserMedia)
|
||||
Dim mediaResult As Boolean
|
||||
|
||||
If IsSavedPosts Then
|
||||
path = $"/api2/v2/posts/bookmarks/all/?format=infinite&limit=10&offset={Cursor}"
|
||||
Else
|
||||
If ID.IsEmptyString Then GetUserID()
|
||||
If ID.IsEmptyString Then Throw New ArgumentNullException("ID", "Unable to get user ID")
|
||||
|
||||
path = $"/api2/v2/users/{ID}/posts/medias?limit=50&order=publish_date_desc&skip_users=all&format=infinite&counters=1"
|
||||
If Not Cursor.IsEmptyString Then path &= $"&counters=0&beforePublishTime={Cursor}" Else path &= "&counters=1"
|
||||
End If
|
||||
|
||||
If UpdateSignature(path) Then
|
||||
url = String.Format(BaseUrlPattern, path)
|
||||
ThrowAny(Token)
|
||||
|
||||
Dim r$ = Responser.GetResponse(url)
|
||||
If Not r.IsEmptyString Then
|
||||
Using j As EContainer = JsonDocument.Parse(r)
|
||||
If j.ListExists Then
|
||||
If IsSavedPosts Then
|
||||
hasMore = j.Value("hasMore").FromXML(Of Boolean)(False)
|
||||
Else
|
||||
tmpCursor = j.Value("tailMarker")
|
||||
hasMore = Not tmpCursor.IsEmptyString
|
||||
End If
|
||||
With j("list")
|
||||
If .ListExists Then
|
||||
ProgressPre.ChangeMax(.Count)
|
||||
For Each n In .Self
|
||||
ProgressPre.Perform()
|
||||
postID = n.Value("id")
|
||||
postDate = n.Value("postedAt")
|
||||
|
||||
If Not _TempPostsList.Contains(postID) Then
|
||||
_TempPostsList.Add(postID)
|
||||
Else
|
||||
Exit Sub
|
||||
End If
|
||||
|
||||
Select Case MyBase.CheckDatesLimit(postDate, DateProvider)
|
||||
Case DateResult.Skip : Continue For
|
||||
Case DateResult.Exit : Exit Sub
|
||||
End Select
|
||||
|
||||
mediaResult = False
|
||||
mediaList = TryCreateMedia(n, postID, postDate, mediaResult)
|
||||
If mediaResult Then _TempMediaList.ListAddList(mediaList, LNC)
|
||||
Next
|
||||
Else
|
||||
hasMore = False
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
End Using
|
||||
End If
|
||||
End If
|
||||
|
||||
If hasMore Then
|
||||
If IsSavedPosts Then tmpCursor = CInt(Cursor.IfNullOrEmpty(0)) + 10
|
||||
DownloadData(tmpCursor, Token)
|
||||
End If
|
||||
Catch ex As Exception
|
||||
If ProcessException(ex, Token, $"data downloading error [{url}]") = 2 Then _complete = False
|
||||
End Try
|
||||
Loop While Not _complete
|
||||
End Sub
|
||||
Private Function TryCreateMedia(ByVal n As EContainer, ByVal PostID As String, Optional ByVal PostDate As String = Nothing,
|
||||
Optional ByRef Result As Boolean = False) As List(Of UserMedia)
|
||||
Dim postUrl$, ext$
|
||||
Dim t As UTypes
|
||||
Dim mList As New List(Of UserMedia)
|
||||
Result = False
|
||||
With n("media")
|
||||
If .ListExists Then
|
||||
For Each m In .Self
|
||||
postUrl = m.Value({"source"}, "source").IfNullOrEmpty(m.Value("full"))
|
||||
Select Case m.Value("type")
|
||||
Case "photo" : t = UTypes.Picture : ext = "jpg"
|
||||
Case "video" : t = UTypes.Video : ext = "mp4"
|
||||
Case Else : t = UTypes.Undefined : ext = String.Empty
|
||||
End Select
|
||||
If Not t = UTypes.Undefined And Not postUrl.IsEmptyString Then
|
||||
Dim media As New UserMedia(postUrl, t) With {
|
||||
.Post = New UserPost(PostID, AConvert(Of Date)(PostDate, DateProvider, Nothing))}
|
||||
media.File.Extension = ext
|
||||
Result = True
|
||||
mList.Add(media)
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
End With
|
||||
Return mList
|
||||
End Function
|
||||
Private Sub GetUserID()
|
||||
Dim path$ = $"/api2/v2/users/{Name}"
|
||||
Dim url$ = String.Format(BaseUrlPattern, path)
|
||||
Try
|
||||
If ID.IsEmptyString AndAlso UpdateSignature(path) Then
|
||||
Dim r$ = Responser.GetResponse(url)
|
||||
If Not r.IsEmptyString Then
|
||||
Using j As EContainer = JsonDocument.Parse(r)
|
||||
If j.ListExists Then
|
||||
ID = j.Value("id")
|
||||
If Not ID.IsEmptyString Then _ForceSaveUserInfo = True
|
||||
UserSiteNameUpdate(j.Value("name"))
|
||||
UserDescriptionUpdate(j.Value("about"))
|
||||
Dim a As Action(Of String) = Sub(ByVal address As String)
|
||||
If Not address.IsEmptyString Then
|
||||
Dim f As SFile = address
|
||||
f.Separator = "\"
|
||||
f.Path = DownloadContentDefault_GetRootDir()
|
||||
If Not f.Exists Then GetWebFile(address, f, EDP.None)
|
||||
End If
|
||||
End Sub
|
||||
a.Invoke(j.Value("avatar"))
|
||||
a.Invoke(j.Value("header"))
|
||||
End If
|
||||
End Using
|
||||
End If
|
||||
End If
|
||||
Catch ex As Exception
|
||||
ProcessException(ex, Nothing, $"user info parsing error [{url}]")
|
||||
End Try
|
||||
End Sub
|
||||
Protected Overrides Sub ReparseMissing(ByVal Token As CancellationToken)
|
||||
Const PathPattern$ = "/api2/v2/posts/{0}?skip_users=all"
|
||||
Dim rList As New List(Of Integer)
|
||||
Dim URL$ = String.Empty
|
||||
Try
|
||||
If ContentMissingExists Then
|
||||
Dim m As UserMedia
|
||||
Dim mList As List(Of UserMedia)
|
||||
Dim mediaResult As Boolean
|
||||
Dim r$, path$, postDate$
|
||||
Dim j As EContainer
|
||||
ProgressPre.ChangeMax(_ContentList.Count)
|
||||
For i% = 0 To _ContentList.Count - 1
|
||||
ProgressPre.Perform()
|
||||
If _ContentList(i).State = UStates.Missing Then
|
||||
m = _ContentList(i)
|
||||
If Not m.Post.ID.IsEmptyString Then
|
||||
ThrowAny(Token)
|
||||
path = String.Format(PathPattern, m.Post.ID)
|
||||
If UpdateSignature(path) Then
|
||||
URL = String.Format(BaseUrlPattern, path)
|
||||
r = Responser.GetResponse(URL,, EDP.ReturnValue)
|
||||
If Not r.IsEmptyString Then
|
||||
j = JsonDocument.Parse(r)
|
||||
If Not j Is Nothing Then
|
||||
postDate = j.Value("postedAt")
|
||||
mediaResult = False
|
||||
mList = TryCreateMedia(j, m.Post.ID, postDate, mediaResult)
|
||||
If mediaResult Then
|
||||
_TempMediaList.ListAddList(mList, LNC)
|
||||
rList.Add(i)
|
||||
mList.Clear()
|
||||
End If
|
||||
j.Dispose()
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
Catch ex As Exception
|
||||
ProcessException(ex, Token, $"ReparseMissing error [{URL}]")
|
||||
Finally
|
||||
If rList.Count > 0 Then
|
||||
For i% = rList.Count - 1 To 0 Step -1 : _ContentList.RemoveAt(i) : Next
|
||||
rList.Clear()
|
||||
End If
|
||||
End Try
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "DownloadSingleObject"
|
||||
Protected Overrides Sub DownloadSingleObject_GetPosts(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken)
|
||||
Dim postID$ = RegexReplace(Data.URL, RegExPostID)
|
||||
If Not postID.IsEmptyString Then _ContentList.Add(New UserMedia With {.Post = postID, .State = UStates.Missing}) : ReparseMissing(Token)
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "Auth"
|
||||
Private ReadOnly Property AuthFile As SFile
|
||||
Get
|
||||
Dim f As SFile = MySettings.Responser.File
|
||||
f.Name &= "_Auth"
|
||||
f.Extension = "json"
|
||||
Return f
|
||||
End Get
|
||||
End Property
|
||||
Private Function UpdateSignature(ByVal Path As String, Optional ByVal ForceUpdateAuth As Boolean = False) As Boolean
|
||||
Try
|
||||
If UpdateAuthFile(ForceUpdateAuth) Then
|
||||
Const nullMsg$ = "The auth parameter is null"
|
||||
Dim j As EContainer = JsonDocument.Parse(AuthFile.GetText)
|
||||
Dim pattern$ = j.Value("format")
|
||||
If pattern.IsEmptyString Then Throw New ArgumentNullException("format", nullMsg)
|
||||
pattern = pattern.Replace("{}", "{0}").Replace("{:x}", "{1:x}")
|
||||
|
||||
Dim li%() = j("checksum_indexes").Select(Function(e) CInt(e(0).Value)).ToArray
|
||||
|
||||
If Not li.ListExists Then Throw New ArgumentNullException("checksum_indexes", nullMsg)
|
||||
If j.Value("static_param").IsEmptyString Then Throw New ArgumentNullException("static_param", nullMsg)
|
||||
If j.Value("checksum_constant").IsEmptyString Then Throw New ArgumentNullException("checksum_constant", nullMsg)
|
||||
|
||||
Dim t$ = ADateTime.ConvertToUnix64(Now.ToUniversalTime).ToString
|
||||
Dim h$ = String.Join(vbLf, j.Value("static_param"), t, Path, MySettings.HH_USER_ID.Value.ToString)
|
||||
|
||||
Dim hash$ = GetHashSha1(h)
|
||||
Dim hashBytes() As Byte = System.Text.Encoding.ASCII.GetBytes(hash)
|
||||
Dim hashSum% = li.Sum(Function(i) hashBytes(i)) + CInt(j.Value("checksum_constant"))
|
||||
Dim sign$ = String.Format(pattern, hash, Math.Abs(hashSum))
|
||||
|
||||
'#If DEBUG Then
|
||||
'Debug.WriteLine(sign)
|
||||
'Debug.WriteLine(t)
|
||||
'#End If
|
||||
|
||||
Responser.Headers.Add(HeaderSign, sign)
|
||||
Responser.Headers.Add(HeaderTime, t)
|
||||
|
||||
j.Dispose()
|
||||
Return True
|
||||
Else
|
||||
Return False
|
||||
End If
|
||||
Catch ex As Exception
|
||||
Return ErrorsDescriber.Execute(EDP.SendToLog + EDP.ReturnValue, ex, $"{ToStringForLog()}: UpdateSignature", False)
|
||||
End Try
|
||||
End Function
|
||||
Private Function UpdateAuthFile(ByVal Force As Boolean) As Boolean
|
||||
Const urlOld$ = "https://raw.githubusercontent.com/DATAHOARDERS/dynamic-rules/main/onlyfans.json"
|
||||
Const urlNew$ = "https://raw.githubusercontent.com/DIGITALCRIMINALS/dynamic-rules/main/onlyfans.json"
|
||||
Try
|
||||
If MySettings.LastDateUpdated.AddMinutes(CInt(MySettings.DynamicRulesUpdateInterval.Value)) < Now Or Not AuthFile.Exists Or Force Then
|
||||
Dim r$ = GetWebString(If(ACheck(Of String)(MySettings.DynamicRules.Value),
|
||||
CStr(MySettings.DynamicRules.Value),
|
||||
IIf(MySettings.UseOldAuthRules.Value, urlOld, urlNew)),, EDP.ReturnValue)
|
||||
If Not r.IsEmptyString Then
|
||||
Using j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue)
|
||||
If j.ListExists Then
|
||||
If Not j.Value("format").IsEmptyString And j("checksum_indexes").ListExists And
|
||||
Not j.Value("static_param").IsEmptyString And Not j.Value("checksum_constant").IsEmptyString Then _
|
||||
TextSaver.SaveTextToFile(r, AuthFile, True, False, EDP.ThrowException) : MySettings.LastDateUpdated = Now
|
||||
End If
|
||||
End Using
|
||||
End If
|
||||
End If
|
||||
Return AuthFile.Exists
|
||||
Catch ex As Exception
|
||||
Return ErrorsDescriber.Execute(EDP.SendToLog + EDP.ReturnValue, ex, $"{ToStringForLog()}: UpdateAuthFile", False)
|
||||
End Try
|
||||
End Function
|
||||
Private Function GetHashSha1(ByVal Input As String) As String
|
||||
Dim s As New Security.Cryptography.SHA1CryptoServiceProvider
|
||||
Dim inputBytes() As Byte = System.Text.Encoding.UTF8.GetBytes(Input)
|
||||
Dim hashBytes() As Byte = s.ComputeHash(inputBytes)
|
||||
s.Dispose()
|
||||
Dim result As String = String.Empty
|
||||
For Each b As Byte In hashBytes : result &= b.ToString("x2") : Next
|
||||
Return result
|
||||
End Function
|
||||
#End Region
|
||||
#Region "DownloadContent"
|
||||
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
|
||||
DownloadContentDefault(Token)
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "DownloadingException"
|
||||
Private _DownloadingException_AuthFileUpdate As Boolean = False
|
||||
Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False,
|
||||
Optional ByVal EObj As Object = Nothing) As Integer
|
||||
If Responser.StatusCode = Net.HttpStatusCode.BadRequest Then
|
||||
If Not _DownloadingException_AuthFileUpdate AndAlso UpdateAuthFile(True) Then
|
||||
_DownloadingException_AuthFileUpdate = True
|
||||
Return 2
|
||||
Else
|
||||
MySettings.SessionAborted = True
|
||||
MyMainLOG = $"{ToStringForLog()}: OnlyFans credentials expired"
|
||||
Return 1
|
||||
End If
|
||||
ElseIf Responser.StatusCode = Net.HttpStatusCode.NotFound Then
|
||||
UserExists = False
|
||||
Return 1
|
||||
Else
|
||||
Return 0
|
||||
End If
|
||||
End Function
|
||||
#End Region
|
||||
#Region "IDisposable Support"
|
||||
Protected Overrides Sub Dispose(ByVal disposing As Boolean)
|
||||
If Not disposedValue And disposing Then CCookie.DisposeIfReady(False) : CCookie = Nothing
|
||||
MyBase.Dispose(disposing)
|
||||
End Sub
|
||||
#End Region
|
||||
End Class
|
||||
End Namespace
|
||||
Reference in New Issue
Block a user