geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-14 15:52:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update names

Browse Source 
Updated library objects
This commit is contained in:
Andy
2022-12-24 15:45:12 +03:00
parent f0686bbc8e
commit 03487185c5
24 changed files with 80 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
SCrawler/API/Base/M3U8Base.vb
View File

@@ -28,7 +28,7 @@ Namespace API.Base
Return $"{Appender.StringTrimEnd("/")}/{File}" Return $"{Appender.StringTrimEnd("/")}/{File}"
End If End If
End Function End Function
Friend Shared Function Download(ByVal URLs As List(Of String), ByVal DestinationFile As SFile, Optional ByVal Responser As Response = Nothing) As SFile Friend Shared Function Download(ByVal URLs As List(Of String), ByVal DestinationFile As SFile, Optional ByVal Responser As Responser = Nothing) As SFile
Dim CachePath As SFile = Nothing Dim CachePath As SFile = Nothing
Try Try
If URLs.ListExists Then If URLs.ListExists Then

6
SCrawler/API/Base/SiteSettingsBase.vb
View File

@@ -17,8 +17,8 @@ Namespace API.Base
Friend Overridable ReadOnly Property Icon As Icon Implements ISiteSettings.Icon Friend Overridable ReadOnly Property Icon As Icon Implements ISiteSettings.Icon
Friend Overridable ReadOnly Property Image As Image Implements ISiteSettings.Image Friend Overridable ReadOnly Property Image As Image Implements ISiteSettings.Image
Private Property Logger As ILogProvider = LogConnector Implements ISiteSettings.Logger Private Property Logger As ILogProvider = LogConnector Implements ISiteSettings.Logger
Friend Overridable ReadOnly Property Responser As Response Friend Overridable ReadOnly Property Responser As Responser
Private Property IResponserContainer_Responser As Response Implements IResponserContainer.Responser Private Property IResponserContainer_Responser As Responser Implements IResponserContainer.Responser
Get Get
Return Responser Return Responser
End Get End Get
@@ -30,7 +30,7 @@ Namespace API.Base
End Sub End Sub
Friend Sub New(ByVal SiteName As String, ByVal CookiesDomain As String) Friend Sub New(ByVal SiteName As String, ByVal CookiesDomain As String)
Site = SiteName Site = SiteName
Responser = New Response($"{SettingsFolderName}\Responser_{Site}.xml") Responser = New Responser($"{SettingsFolderName}\Responser_{Site}.xml")
With Responser With Responser
If .File.Exists Then If .File.Exists Then
If EncryptCookies.CookiesEncrypted Then .CookiesEncryptKey = SettingsCLS.CookieEncryptKey If EncryptCookies.CookiesEncrypted Then .CookiesEncryptKey = SettingsCLS.CookieEncryptKey

4
SCrawler/API/Base/UserDataBase.vb
View File

@@ -843,7 +843,7 @@ BlockNullPicture:
End Function End Function
#End Region #End Region
#Region "Download functions and options" #Region "Download functions and options"
Protected Responser As Response Protected Responser As Responser
Protected UseResponserClient As Boolean = False Protected UseResponserClient As Boolean = False
Friend Overridable Sub DownloadData(ByVal Token As CancellationToken) Implements IContentProvider.DownloadData Friend Overridable Sub DownloadData(ByVal Token As CancellationToken) Implements IContentProvider.DownloadData
Dim Canceled As Boolean = False Dim Canceled As Boolean = False
@@ -852,7 +852,7 @@ BlockNullPicture:
UpdateDataFiles() UpdateDataFiles()
UserDescriptionReset() UserDescriptionReset()
If Not Responser Is Nothing Then Responser.Dispose() If Not Responser Is Nothing Then Responser.Dispose()
Responser = New Response Responser = New Responser
If Not HOST.Responser Is Nothing Then Responser.Copy(HOST.Responser) If Not HOST.Responser Is Nothing Then Responser.Copy(HOST.Responser)
'TODO: UserDataBase remove [Responser.DecodersError] 'TODO: UserDataBase remove [Responser.DecodersError]
Responser.DecodersError = New ErrorsDescriber(EDP.SendInLog + EDP.ReturnValue) With { Responser.DecodersError = New ErrorsDescriber(EDP.SendInLog + EDP.ReturnValue) With {

16
SCrawler/API/Instagram/SiteSettings.vb
View File

@@ -106,8 +106,8 @@ Namespace API.Instagram
Case NameOf(CSRF_TOKEN) : f = Header_CSRF_TOKEN Case NameOf(CSRF_TOKEN) : f = Header_CSRF_TOKEN
End Select End Select
If Not f.IsEmptyString Then If Not f.IsEmptyString Then
Responser.HeadersRemove(f) Responser.Headers.Remove(f)
If Not CStr(Value).IsEmptyString Then Responser.HeadersAdd(f, CStr(Value)) If Not CStr(Value).IsEmptyString Then Responser.Headers.Add(f, CStr(Value))
Responser.SaveSettings() Responser.SaveSettings()
End If End If
End If End If
@@ -202,9 +202,9 @@ Namespace API.Instagram
With Responser With Responser
If .Headers.Count > 0 Then If .Headers.Count > 0 Then
token = .HeadersValue(Header_CSRF_TOKEN) token = .Headers.Value(Header_CSRF_TOKEN)
app_id = .HeadersValue(Header_IG_APP_ID) app_id = .Headers.Value(Header_IG_APP_ID)
www_claim = .HeadersValue(Header_IG_WWW_CLAIM) www_claim = .Headers.Value(Header_IG_WWW_CLAIM)
End If End If
If Not .Cookies Is Nothing Then If Not .Cookies Is Nothing Then
.Cookies.ChangedAllowInternalDrop = False .Cookies.ChangedAllowInternalDrop = False
@@ -262,9 +262,9 @@ Namespace API.Instagram
Hash = AConvert(Of String)(.Hash.Value, String.Empty) Hash = AConvert(Of String)(.Hash.Value, String.Empty)
Hash2 = AConvert(Of String)(.HashSavedPosts.Value, String.Empty) Hash2 = AConvert(Of String)(.HashSavedPosts.Value, String.Empty)
With .Responser With .Responser
Token = .HeadersValue(Header_CSRF_TOKEN) Token = .Headers.Value(Header_CSRF_TOKEN)
AppID = .HeadersValue(Header_IG_APP_ID) AppID = .Headers.Value(Header_IG_APP_ID)
WwwClaim = .HeadersValue(Header_IG_WWW_CLAIM) WwwClaim = .Headers.Value(Header_IG_WWW_CLAIM)
End With End With
End With End With
End Sub End Sub

4
SCrawler/API/Instagram/UserData.vb
View File

@@ -750,7 +750,7 @@ Namespace API.Instagram
End Function End Function
#End Region #End Region
#Region "Standalone downloader" #Region "Standalone downloader"
Friend Shared Function GetVideoInfo(ByVal URL As String, ByVal r As Response) As IEnumerable(Of UserMedia) Friend Shared Function GetVideoInfo(ByVal URL As String, ByVal r As Responser) As IEnumerable(Of UserMedia)
Try Try
If Not URL.IsEmptyString AndAlso URL.Contains("instagram.com") Then If Not URL.IsEmptyString AndAlso URL.Contains("instagram.com") Then
Dim PID$ = RegexReplace(URL, RParams.DMS(".*?instagram.com/p/([_\w\d]+)", 1)) Dim PID$ = RegexReplace(URL, RParams.DMS(".*?instagram.com/p/([_\w\d]+)", 1))
@@ -758,7 +758,7 @@ Namespace API.Instagram
If Not PID.IsEmptyString Then If Not PID.IsEmptyString Then
Using t As New UserData Using t As New UserData
t.SetEnvironment(Settings(InstagramSiteKey), Nothing, False, False) t.SetEnvironment(Settings(InstagramSiteKey), Nothing, False, False)
t.Responser = New Response t.Responser = New Responser
t.Responser.Copy(r) t.Responser.Copy(r)
t._SavedPostsIDs.Add(PID) t._SavedPostsIDs.Add(PID)
t.DownloadPosts(Nothing) t.DownloadPosts(Nothing)

2
SCrawler/API/LPSG/UserData.vb
View File

@@ -88,7 +88,7 @@ Namespace API.LPSG
End If End If
End Sub End Sub
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken) Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
With Responser : .Mode = Response.Modes.WebClient : .ResetStatus() : End With With Responser : .Mode = Responser.Modes.WebClient : .ResetStatus() : End With
UseResponserClient = True UseResponserClient = True
DownloadContentDefault(Token) DownloadContentDefault(Token)
End Sub End Sub

4
SCrawler/API/PornHub/M3U8.vb
View File

@@ -14,7 +14,7 @@ Namespace API.PornHub
Friend NotInheritable Class M3U8 Friend NotInheritable Class M3U8
Private Sub New() Private Sub New()
End Sub End Sub
Private Shared Function GetUrlsList(ByVal URL As String, ByVal Responser As Response) As List(Of String) Private Shared Function GetUrlsList(ByVal URL As String, ByVal Responser As Responser) As List(Of String)
Dim appender$ = RegexReplace(URL, Regex_M3U8_FileUrl) Dim appender$ = RegexReplace(URL, Regex_M3U8_FileUrl)
Dim r$ = Responser.GetResponse(URL) Dim r$ = Responser.GetResponse(URL)
If Not r.IsEmptyString Then If Not r.IsEmptyString Then
@@ -35,7 +35,7 @@ Namespace API.PornHub
End If End If
Return Nothing Return Nothing
End Function End Function
Friend Shared Function Download(ByVal URL As String, ByVal Responser As Response, ByVal Destination As SFile) As SFile Friend Shared Function Download(ByVal URL As String, ByVal Responser As Responser, ByVal Destination As SFile) As SFile
Return M3U8Base.Download(GetUrlsList(URL, Responser), Destination, Responser) Return M3U8Base.Download(GetUrlsList(URL, Responser), Destination, Responser)
End Function End Function
End Class End Class

2
SCrawler/API/PornHub/SiteSettings.vb
View File

@@ -69,7 +69,7 @@ Namespace API.PornHub
End Function End Function
Friend Overrides Function GetSpecialData(ByVal URL As String, ByVal Path As String, ByVal AskForPath As Boolean) As IEnumerable Friend Overrides Function GetSpecialData(ByVal URL As String, ByVal Path As String, ByVal AskForPath As Boolean) As IEnumerable
If Available(ISiteSettings.Download.Main, True) Then If Available(ISiteSettings.Download.Main, True) Then
Using resp As Response = Responser.Copy Using resp As Responser = Responser.Copy
Dim spf$ = String.Empty Dim spf$ = String.Empty
Dim f As SFile = GetSpecialDataFile(Path, AskForPath, spf) Dim f As SFile = GetSpecialDataFile(Path, AskForPath, spf)
Dim m As UserMedia = UserData.GetVideoInfo(URL, resp, f) Dim m As UserMedia = UserData.GetVideoInfo(URL, resp, f)

8
SCrawler/API/PornHub/UserData.vb
View File

@@ -178,7 +178,7 @@ Namespace API.PornHub
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken) Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
Try Try
Responser.ResetStatus() Responser.ResetStatus()
If PersonType = PersonTypeUser Then Responser.Mode = Response.Modes.Curl If PersonType = PersonTypeUser Then Responser.Mode = Responser.Modes.Curl
If IsSavedPosts Then VideoPageModel = VideoPageModels.Favorite If IsSavedPosts Then VideoPageModel = VideoPageModels.Favorite
@@ -187,7 +187,7 @@ Namespace API.PornHub
Dim __videoDone As Boolean = False Dim __videoDone As Boolean = False
Dim d% Dim d%
If DownloadVideos Then If DownloadVideos Then
If PersonType = PersonTypeUser Then Responser.Mode = Response.Modes.Curl : Responser.Method = "POST" If PersonType = PersonTypeUser Then Responser.Mode = Responser.Modes.Curl : Responser.Method = "POST"
If VideoPageModel = VideoPageModels.Undefined Then If VideoPageModel = VideoPageModels.Undefined Then
__continue = False __continue = False
d = DownloadUserVideos(page, Token) d = DownloadUserVideos(page, Token)
@@ -216,7 +216,7 @@ Namespace API.PornHub
If DownloadGifs And Not IsSavedPosts Then DownloadUserGifs(Token) If DownloadGifs And Not IsSavedPosts Then DownloadUserGifs(Token)
If DownloadImages Then DownloadUserPhotos(Token) If DownloadImages Then DownloadUserPhotos(Token)
Finally Finally
Responser.Mode = Response.Modes.Default Responser.Mode = Responser.Modes.Default
Responser.Method = "GET" Responser.Method = "GET"
End Try End Try
End Sub End Sub
@@ -624,7 +624,7 @@ Namespace API.PornHub
End Function End Function
#End Region #End Region
#Region "Standalone downloader" #Region "Standalone downloader"
Friend Shared Function GetVideoInfo(ByVal URL As String, ByVal Responser As Response, ByVal Destination As SFile) As UserMedia Friend Shared Function GetVideoInfo(ByVal URL As String, ByVal Responser As Responser, ByVal Destination As SFile) As UserMedia
Try Try
Dim r$ = Responser.Curl(URL) Dim r$ = Responser.Curl(URL)
If Not r.IsEmptyString Then If Not r.IsEmptyString Then

12
SCrawler/API/Reddit/UserData.vb
View File

@@ -136,7 +136,7 @@ Namespace API.Reddit
_CrossPosts.Clear() _CrossPosts.Clear()
If Not IsSavedPosts AndAlso (IsChannel AndAlso Not ChannelInfo Is Nothing) Then If Not IsSavedPosts AndAlso (IsChannel AndAlso Not ChannelInfo Is Nothing) Then
If Not Responser Is Nothing Then Responser.Dispose() If Not Responser Is Nothing Then Responser.Dispose()
Responser = New Response Responser = New Responser
Responser.Copy(MySiteSettings.Responser) Responser.Copy(MySiteSettings.Responser)
ChannelPostsNames.ListAddList(ChannelInfo.PostsAll.Select(Function(p) p.ID), LNC) ChannelPostsNames.ListAddList(ChannelInfo.PostsAll.Select(Function(p) p.ID), LNC)
If Not ViewMode = CView.New Then ChannelPostsNames.ListAddList(ChannelInfo.PostsNames, LNC) If Not ViewMode = CView.New Then ChannelPostsNames.ListAddList(ChannelInfo.PostsNames, LNC)
@@ -524,7 +524,7 @@ Namespace API.Reddit
End Try End Try
End Function End Function
Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken) Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken)
Dim RedGifsResponser As Response = Nothing Dim RedGifsResponser As Responser = Nothing
Try Try
ThrowAny(Token) ThrowAny(Token)
Const v2 As UTypes = UTypes.VideoPre + UTypes.m3u8 Const v2 As UTypes = UTypes.VideoPre + UTypes.m3u8
@@ -581,7 +581,7 @@ Namespace API.Reddit
End Sub End Sub
Protected Overrides Sub ReparseMissing(ByVal Token As CancellationToken) Protected Overrides Sub ReparseMissing(ByVal Token As CancellationToken)
Dim rList As New List(Of Integer) Dim rList As New List(Of Integer)
Dim RedGifsResponser As Response = Nothing Dim RedGifsResponser As Responser = Nothing
Try Try
If Not ChannelInfo Is Nothing Or SaveToCache Then Exit Sub If Not ChannelInfo Is Nothing Or SaveToCache Then Exit Sub
If ContentMissingExists Then If ContentMissingExists Then
@@ -658,11 +658,11 @@ Namespace API.Reddit
Public Overrides Sub Perform(Optional ByVal Value As Double = 1) Public Overrides Sub Perform(Optional ByVal Value As Double = 1)
End Sub End Sub
End Class End Class
Friend Shared Function GetVideoInfo(ByVal URL As String, ByVal resp As Response, ByVal f As SFile, ByVal SpecialFolder As String) As IEnumerable(Of UserMedia) Friend Shared Function GetVideoInfo(ByVal URL As String, ByVal resp As Responser, ByVal f As SFile, ByVal SpecialFolder As String) As IEnumerable(Of UserMedia)
Try Try
If Not URL.IsEmptyString Then If Not URL.IsEmptyString Then
Using r As New UserData Using r As New UserData
r.Responser = New Response r.Responser = New Responser
r.Responser.Copy(resp) r.Responser.Copy(resp)
r.ParsePost(URL) r.ParsePost(URL)
If r._TempMediaList.Count > 0 Then If r._TempMediaList.Count > 0 Then
@@ -713,7 +713,7 @@ Namespace API.Reddit
End Function End Function
#End Region #End Region
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken) Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
Dim RedGifsResponser As Response = Nothing Dim RedGifsResponser As Responser = Nothing
Try Try
Const _RFN$ = "RedditVideo" Const _RFN$ = "RedditVideo"
Const RFN$ = _RFN & "{0}" Const RFN$ = _RFN & "{0}"

16
SCrawler/API/Redgifs/SiteSettings.vb
View File

@@ -33,7 +33,7 @@ Namespace API.RedGifs
<PropertyOption(AllowNull:=False, ControlText:="Token", ControlToolTip:="Bearer token")> <PropertyOption(AllowNull:=False, ControlText:="Token", ControlToolTip:="Bearer token")>
Friend Property Token As PropertyValue Friend Property Token As PropertyValue
<PXML> Friend Property TokenLastDateUpdated As PropertyValue <PXML> Friend Property TokenLastDateUpdated As PropertyValue
<DoNotUse> Friend ReadOnly Property NoCredentialsResponser As Response <DoNotUse> Friend ReadOnly Property NoCredentialsResponser As Responser
Private Const TokenName As String = "authorization" Private Const TokenName As String = "authorization"
#End Region #End Region
#Region "Initializer" #Region "Initializer"
@@ -41,12 +41,12 @@ Namespace API.RedGifs
MyBase.New(RedGifsSite, "redgifs.com") MyBase.New(RedGifsSite, "redgifs.com")
Dim t$ = String.Empty Dim t$ = String.Empty
With Responser With Responser
Dim b As Boolean = Not .Mode = Response.Modes.WebClient Dim b As Boolean = Not .Mode = Responser.Modes.WebClient
.Mode = Response.Modes.WebClient .Mode = Responser.Modes.WebClient
t = .HeadersValue(TokenName) t = .Headers.Value(TokenName)
If b Then .SaveSettings() If b Then .SaveSettings()
End With End With
NoCredentialsResponser = New Response($"{SettingsFolderName}\Responser_{RedGifsSite}_NC.xml") With { NoCredentialsResponser = New Responser($"{SettingsFolderName}\Responser_{RedGifsSite}_NC.xml") With {
.CookiesEncryptKey = SettingsCLS.CookieEncryptKey, .CookiesEncryptKey = SettingsCLS.CookieEncryptKey,
.CookiesDomain = "redgifs.com" .CookiesDomain = "redgifs.com"
} }
@@ -67,7 +67,7 @@ Namespace API.RedGifs
#End Region #End Region
#Region "Response updater" #Region "Response updater"
Private Sub UpdateResponse(ByVal Value As String) Private Sub UpdateResponse(ByVal Value As String)
Responser.HeadersAdd(TokenName, Value) Responser.Headers.Add(TokenName, Value)
Responser.SaveSettings() Responser.SaveSettings()
End Sub End Sub
#End Region #End Region
@@ -85,7 +85,7 @@ Namespace API.RedGifs
Try Try
Dim r$ Dim r$
Dim NewToken$ = String.Empty Dim NewToken$ = String.Empty
Using resp As New Response : r = resp.GetResponse("https://api.redgifs.com/v2/auth/temporary",, EDP.ThrowException) : End Using Using resp As New Responser : r = resp.GetResponse("https://api.redgifs.com/v2/auth/temporary",, EDP.ThrowException) : End Using
If Not r.IsEmptyString Then If Not r.IsEmptyString Then
Dim j As EContainer = JsonDocument.Parse(r) Dim j As EContainer = JsonDocument.Parse(r)
If Not j Is Nothing Then If Not j Is Nothing Then
@@ -126,7 +126,7 @@ Namespace API.RedGifs
End Function End Function
Friend Overrides Function GetSpecialData(ByVal URL As String, ByVal Path As String, ByVal AskForPath As Boolean) As IEnumerable Friend Overrides Function GetSpecialData(ByVal URL As String, ByVal Path As String, ByVal AskForPath As Boolean) As IEnumerable
If BaseAuthExists() Then If BaseAuthExists() Then
Using resp As Response = Responser.Copy Using resp As Responser = Responser.Copy
Dim m As UserMedia = UserData.GetDataFromUrlId(URL, False, resp, Settings(RedGifsSiteKey)) Dim m As UserMedia = UserData.GetDataFromUrlId(URL, False, resp, Settings(RedGifsSiteKey))
If Not m.State = UStates.Missing And Not m.State = UserData.DataGone And (m.Type = UTypes.Picture Or m.Type = UTypes.Video) Then If Not m.State = UStates.Missing And Not m.State = UserData.DataGone And (m.Type = UTypes.Picture Or m.Type = UTypes.Video) Then
Try Try

4
SCrawler/API/Redgifs/UserData.vb
View File

@@ -34,7 +34,7 @@ Namespace API.RedGifs
End Sub End Sub
#End Region #End Region
#Region "Download functions" #Region "Download functions"
Private NoCredentialsResponser As Response Private NoCredentialsResponser As Responser
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken) Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
Try Try
NoCredentialsResponser = MySettings.NoCredentialsResponser.Copy NoCredentialsResponser = MySettings.NoCredentialsResponser.Copy
@@ -166,7 +166,7 @@ Namespace API.RedGifs
Return String.Empty Return String.Empty
End If End If
End Function End Function
Friend Shared Function GetDataFromUrlId(ByVal Obj As String, ByVal ObjIsID As Boolean, ByVal Responser As Response, Friend Shared Function GetDataFromUrlId(ByVal Obj As String, ByVal ObjIsID As Boolean, ByVal Responser As Responser,
ByVal Host As Plugin.Hosts.SettingsHost) As UserMedia ByVal Host As Plugin.Hosts.SettingsHost) As UserMedia
Dim URL$ = String.Empty Dim URL$ = String.Empty
Try Try

4
SCrawler/API/TikTok/UserData.vb
View File

@@ -52,7 +52,7 @@ Namespace API.TikTok
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken) Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
DownloadContentDefault(Token) DownloadContentDefault(Token)
End Sub End Sub
Friend Shared Function GetVideoInfo(ByVal URL As String, ByVal Responser As Response, Optional ByVal e As ErrorsDescriber = Nothing) As IEnumerable(Of UserMedia) Friend Shared Function GetVideoInfo(ByVal URL As String, ByVal Responser As Responser, Optional ByVal e As ErrorsDescriber = Nothing) As IEnumerable(Of UserMedia)
Try Try
If Not URL.IsEmptyString Then If Not URL.IsEmptyString Then
Dim PostId$ = String.Empty Dim PostId$ = String.Empty
@@ -61,7 +61,7 @@ Namespace API.TikTok
Dim r$ Dim r$
PostId = RegexEnvir.ExtractPostID(URL) PostId = RegexEnvir.ExtractPostID(URL)
If Not PostId.IsEmptyString Then If Not PostId.IsEmptyString Then
Using resp As Response = Responser.Copy() : r = resp.GetResponse(URL,, EDP.ThrowException) : End Using Using resp As Responser = Responser.Copy() : r = resp.GetResponse(URL,, EDP.ThrowException) : End Using
If Not r.IsEmptyString Then If Not r.IsEmptyString Then
If RegexEnvir.GetVideoData(r, PostId, PostURL, PostDate) Then Return {MediaFromData(PostURL, PostId, PostDate)} If RegexEnvir.GetVideoData(r, PostId, PostURL, PostDate) Then Return {MediaFromData(PostURL, PostId, PostDate)}
End If End If

30
SCrawler/API/Twitter/SiteSettings.vb
View File

@@ -34,10 +34,10 @@ Namespace API.Twitter
Private ReadOnly Property Token As PropertyValue Private ReadOnly Property Token As PropertyValue
<PropertyOption(ControlText:="Saved posts user", ControlToolTip:="Personal profile username"), PXML> <PropertyOption(ControlText:="Saved posts user", ControlToolTip:="Personal profile username"), PXML>
Friend ReadOnly Property SavedPostsUserName As PropertyValue Friend ReadOnly Property SavedPostsUserName As PropertyValue
Friend Overrides ReadOnly Property Responser As Response Friend Overrides ReadOnly Property Responser As Responser
Friend Sub New() Friend Sub New()
MyBase.New(TwitterSite) MyBase.New(TwitterSite)
Responser = New Response($"{SettingsFolderName}\Responser_{Site}.xml") Responser = New Responser($"{SettingsFolderName}\Responser_{Site}.xml")
Dim a$ = String.Empty Dim a$ = String.Empty
Dim t$ = String.Empty Dim t$ = String.Empty
@@ -46,8 +46,8 @@ Namespace API.Twitter
If .File.Exists Then If .File.Exists Then
If EncryptCookies.CookiesEncrypted Then .CookiesEncryptKey = SettingsCLS.CookieEncryptKey If EncryptCookies.CookiesEncrypted Then .CookiesEncryptKey = SettingsCLS.CookieEncryptKey
.LoadSettings() .LoadSettings()
a = .HeadersValue(Header_Authorization) a = .Headers.Value(Header_Authorization)
t = .HeadersValue(Header_Token) t = .Headers.Value(Header_Token)
Else Else
.ContentType = "application/json" .ContentType = "application/json"
.Accept = "*/*" .Accept = "*/*"
@@ -55,15 +55,15 @@ Namespace API.Twitter
.Cookies = New CookieKeeper(.CookiesDomain) With {.EncryptKey = SettingsCLS.CookieEncryptKey} .Cookies = New CookieKeeper(.CookiesDomain) With {.EncryptKey = SettingsCLS.CookieEncryptKey}
.CookiesEncryptKey = SettingsCLS.CookieEncryptKey .CookiesEncryptKey = SettingsCLS.CookieEncryptKey
.Decoders.Add(SymbolsConverter.Converters.Unicode) .Decoders.Add(SymbolsConverter.Converters.Unicode)
.HeadersAdd("sec-ch-ua", " Not;A Brand"";v=""99"", ""Google Chrome"";v=""91"", ""Chromium"";v=""91""") .Headers.Add("sec-ch-ua", " Not;A Brand"";v=""99"", ""Google Chrome"";v=""91"", ""Chromium"";v=""91""")
.HeadersAdd("sec-ch-ua-mobile", "?0") .Headers.Add("sec-ch-ua-mobile", "?0")
.HeadersAdd("sec-fetch-dest", "empty") .Headers.Add("sec-fetch-dest", "empty")
.HeadersAdd("sec-fetch-mode", "cors") .Headers.Add("sec-fetch-mode", "cors")
.HeadersAdd("sec-fetch-site", "same-origin") .Headers.Add("sec-fetch-site", "same-origin")
.HeadersAdd(Header_Token, String.Empty) .Headers.Add(Header_Token, String.Empty)
.HeadersAdd("x-twitter-active-user", "yes") .Headers.Add("x-twitter-active-user", "yes")
.HeadersAdd("x-twitter-auth-type", "OAuth2Session") .Headers.Add("x-twitter-auth-type", "OAuth2Session")
.HeadersAdd(Header_Authorization, String.Empty) .Headers.Add(Header_Authorization, String.Empty)
.SaveSettings() .SaveSettings()
End If End If
End With End With
@@ -84,8 +84,8 @@ Namespace API.Twitter
Case NameOf(Token) : f = Header_Token Case NameOf(Token) : f = Header_Token
End Select End Select
If Not f.IsEmptyString Then If Not f.IsEmptyString Then
Responser.HeadersRemove(f) Responser.Headers.Remove(f)
If Not CStr(Value).IsEmptyString Then Responser.HeadersAdd(f, CStr(Value)) If Not CStr(Value).IsEmptyString Then Responser.Headers.Add(f, CStr(Value))
Responser.SaveSettings() Responser.SaveSettings()
End If End If
End If End If

4
SCrawler/API/Twitter/UserData.vb
View File

@@ -263,13 +263,13 @@ Namespace API.Twitter
End Sub End Sub
#End Region #End Region
#Region "Get video static" #Region "Get video static"
Friend Shared Function GetVideoInfo(ByVal URL As String, ByVal resp As Response) As IEnumerable(Of UserMedia) Friend Shared Function GetVideoInfo(ByVal URL As String, ByVal resp As Responser) As IEnumerable(Of UserMedia)
Try Try
If URL.Contains("twitter") Then If URL.Contains("twitter") Then
Dim PostID$ = RegexReplace(URL, RParams.DM("(?<=/)\d+", 0)) Dim PostID$ = RegexReplace(URL, RParams.DM("(?<=/)\d+", 0))
If Not PostID.IsEmptyString Then If Not PostID.IsEmptyString Then
Dim r$ Dim r$
Using rc As Response = resp.Copy() : r = rc.GetResponse(String.Format(SinglePostUrl, PostID),, EDP.ReturnValue) : End Using Using rc As Responser = resp.Copy() : r = rc.GetResponse(String.Format(SinglePostUrl, PostID),, EDP.ReturnValue) : End Using
If Not r.IsEmptyString Then If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r) Using j As EContainer = JsonDocument.Parse(r)
If j.ListExists Then If j.ListExists Then

2
SCrawler/API/XVIDEOS/SiteSettings.vb
View File

@@ -141,7 +141,7 @@ Namespace API.XVIDEOS
Dim f As SFile = GetSpecialDataFile(Path, AskForPath, spf) Dim f As SFile = GetSpecialDataFile(Path, AskForPath, spf)
f.Name = "video" f.Name = "video"
f.Extension = "mp4" f.Extension = "mp4"
Using resp As Response = Responser.Copy Using resp As Responser = Responser.Copy
Using user As New UserData With {.HOST = Settings(XvideosSiteKey)} Using user As New UserData With {.HOST = Settings(XvideosSiteKey)}
DirectCast(user, UserDataBase).User.File = f DirectCast(user, UserDataBase).User.File = f
Dim p As UserMedia = user.Download(URL, resp, DownloadUHD.Value, String.Empty) Dim p As UserMedia = user.Download(URL, resp, DownloadUHD.Value, String.Empty)

4
SCrawler/API/XVIDEOS/UserData.vb
View File

@@ -171,7 +171,7 @@ Namespace API.XVIDEOS
ProcessException(ex, Token, $"data downloading error [{URL}]") ProcessException(ex, Token, $"data downloading error [{URL}]")
End Try End Try
End Sub End Sub
Private Function GetVideoData(ByVal Media As UserMedia, ByVal resp As Response, ByVal DownloadUHD As Boolean) As UserMedia Private Function GetVideoData(ByVal Media As UserMedia, ByVal resp As Responser, ByVal DownloadUHD As Boolean) As UserMedia
Try Try
If Not Media.URL.IsEmptyString Then If Not Media.URL.IsEmptyString Then
Dim r$ = resp.GetResponse(Media.URL) Dim r$ = resp.GetResponse(Media.URL)
@@ -217,7 +217,7 @@ Namespace API.XVIDEOS
Return Nothing Return Nothing
End Try End Try
End Function End Function
Friend Function Download(ByVal URL As String, ByVal resp As Response, ByVal DownloadUHD As Boolean, ByVal ID As String) Friend Function Download(ByVal URL As String, ByVal resp As Responser, ByVal DownloadUHD As Boolean, ByVal ID As String)
Dim m As UserMedia = GetVideoData(New UserMedia(URL, UTypes.VideoPre) With {.Post = ID}, resp, DownloadUHD) Dim m As UserMedia = GetVideoData(New UserMedia(URL, UTypes.VideoPre) With {.Post = ID}, resp, DownloadUHD)
If Not m.URL.IsEmptyString Then If Not m.URL.IsEmptyString Then
Dim f As SFile = m.File Dim f As SFile = m.File

8
SCrawler/API/Xhamster/M3U8.vb
View File

@@ -14,7 +14,7 @@ Namespace API.Xhamster
Friend NotInheritable Class M3U8 Friend NotInheritable Class M3U8
Private Sub New() Private Sub New()
End Sub End Sub
Private Shared Function ParseFirstM3U8(ByVal URL As String, ByVal Responser As Response, ByVal UHD As Boolean) As String Private Shared Function ParseFirstM3U8(ByVal URL As String, ByVal Responser As Responser, ByVal UHD As Boolean) As String
Dim r$, d$ Dim r$, d$
Dim _DataObtained As Boolean = False Dim _DataObtained As Boolean = False
For i% = 0 To 1 For i% = 0 To 1
@@ -38,7 +38,7 @@ Namespace API.Xhamster
Next Next
Return String.Empty Return String.Empty
End Function End Function
Private Shared Function ParseSecondM3U8(ByVal URL As String, ByVal Responser As Response, ByVal Appender As String) As List(Of String) Private Shared Function ParseSecondM3U8(ByVal URL As String, ByVal Responser As Responser, ByVal Appender As String) As List(Of String)
Dim r$ Dim r$
Dim l As List(Of String) Dim l As List(Of String)
For i% = 0 To 1 For i% = 0 To 1
@@ -57,7 +57,7 @@ Namespace API.Xhamster
Next Next
Return Nothing Return Nothing
End Function End Function
Private Shared Function ObtainUrls(ByVal URL As String, ByVal Responser As Response, ByVal UHD As Boolean) As List(Of String) Private Shared Function ObtainUrls(ByVal URL As String, ByVal Responser As Responser, ByVal UHD As Boolean) As List(Of String)
Try Try
Dim file$ = ParseFirstM3U8(URL, Responser, UHD) Dim file$ = ParseFirstM3U8(URL, Responser, UHD)
If Not file.IsEmptyString Then If Not file.IsEmptyString Then
@@ -72,7 +72,7 @@ Namespace API.Xhamster
Responser.UseGZipStream = False Responser.UseGZipStream = False
End Try End Try
End Function End Function
Friend Shared Function Download(ByVal Media As UserMedia, ByVal Responser As Response, ByVal UHD As Boolean) As SFile Friend Shared Function Download(ByVal Media As UserMedia, ByVal Responser As Responser, ByVal UHD As Boolean) As SFile
Return M3U8Base.Download(ObtainUrls(Media.URL, Responser, UHD), Media.File, Responser) Return M3U8Base.Download(ObtainUrls(Media.URL, Responser, UHD), Media.File, Responser)
End Function End Function
End Class End Class

2
SCrawler/API/Xhamster/SiteSettings.vb
View File

@@ -102,7 +102,7 @@ Namespace API.Xhamster
End Function End Function
Friend Overrides Function GetSpecialData(ByVal URL As String, ByVal Path As String, ByVal AskForPath As Boolean) As IEnumerable Friend Overrides Function GetSpecialData(ByVal URL As String, ByVal Path As String, ByVal AskForPath As Boolean) As IEnumerable
If Available(ISiteSettings.Download.Main, True) Then If Available(ISiteSettings.Download.Main, True) Then
Using resp As Response = Responser.Copy Using resp As Responser = Responser.Copy
Dim spf$ = String.Empty Dim spf$ = String.Empty
Dim f As SFile = GetSpecialDataFile(Path, AskForPath, spf) Dim f As SFile = GetSpecialDataFile(Path, AskForPath, spf)
Dim m As UserMedia = UserData.GetVideoInfo(URL, resp, f) Dim m As UserMedia = UserData.GetVideoInfo(URL, resp, f)

4
SCrawler/API/Xhamster/UserData.vb
View File

@@ -219,7 +219,7 @@ Namespace API.Xhamster
End Sub End Sub
#End Region #End Region
#Region "GetM3U8" #Region "GetM3U8"
Private Overloads Function GetM3U8(ByRef m As UserMedia, ByVal URL As String, ByVal Responser As Response, Private Overloads Function GetM3U8(ByRef m As UserMedia, ByVal URL As String, ByVal Responser As Responser,
Optional ByVal e As ErrorsDescriber = Nothing) As Boolean Optional ByVal e As ErrorsDescriber = Nothing) As Boolean
Try Try
If Not URL.IsEmptyString Then If Not URL.IsEmptyString Then
@@ -248,7 +248,7 @@ Namespace API.Xhamster
End Function End Function
#End Region #End Region
#Region "Standalone downloader" #Region "Standalone downloader"
Friend Shared Function GetVideoInfo(ByVal URL As String, ByVal Responser As Response, ByVal Path As SFile) As UserMedia Friend Shared Function GetVideoInfo(ByVal URL As String, ByVal Responser As Responser, ByVal Path As SFile) As UserMedia
Try Try
Using u As New UserData With {.Responser = Responser, .HOST = Settings(XhamsterSiteKey)} Using u As New UserData With {.Responser = Responser, .HOST = Settings(XhamsterSiteKey)}
Dim m As UserMedia = Nothing Dim m As UserMedia = Nothing

4
SCrawler/Download/WebClient2.vb
View File

@@ -11,12 +11,12 @@ Imports PersonalUtilities.Tools.Web.Clients
Namespace DownloadObjects Namespace DownloadObjects
Friend Class WebClient2 : Implements IDisposable Friend Class WebClient2 : Implements IDisposable
Protected WC As WebClient Protected WC As WebClient
Protected RC As Response Protected RC As Responser
Private ReadOnly RCERROR As New ErrorsDescriber(EDP.ThrowException) Private ReadOnly RCERROR As New ErrorsDescriber(EDP.ThrowException)
Protected UseResponserClient As Boolean Protected UseResponserClient As Boolean
Friend Sub New() Friend Sub New()
End Sub End Sub
Friend Sub New(ByVal Responser As Response) Friend Sub New(ByVal Responser As Responser)
If Not Responser Is Nothing Then If Not Responser Is Nothing Then
RC = Responser RC = Responser
UseResponserClient = True UseResponserClient = True

10
SCrawler/Editors/SiteEditorForm.vb
View File

@@ -12,7 +12,6 @@ Imports PersonalUtilities.Forms
Imports PersonalUtilities.Forms.Controls Imports PersonalUtilities.Forms.Controls
Imports PersonalUtilities.Forms.Controls.Base Imports PersonalUtilities.Forms.Controls.Base
Imports PersonalUtilities.Tools.Web.Cookies Imports PersonalUtilities.Tools.Web.Cookies
Imports CookieControl = PersonalUtilities.Tools.Web.Cookies.CookieListForm.CookieControl
Imports ADB = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons Imports ADB = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons
Namespace Editors Namespace Editors
Friend Class SiteEditorForm Friend Class SiteEditorForm
@@ -213,12 +212,13 @@ Namespace Editors
Select Case Sender.DefaultButton Select Case Sender.DefaultButton
Case ADB.Edit Case ADB.Edit
If Not Host.Responser Is Nothing Then If Not Host.Responser Is Nothing Then
Using f As New CookieListForm(Host.Responser) With { Using f As New CookieListForm With {.DesignXML = Settings.Design, .UseGrid = False}
.MyDesignXML = Settings.Design, f.SetCollection(Host.Responser.Cookies)
.DisableControls = CookieControl.AddFromInternal + CookieControl.AuthorizeProgram + CookieControl.OpenBrowser
}
f.ShowDialog() f.ShowDialog()
If f.DialogResult = DialogResult.OK Then
f.GetCollection(Host.Responser)
MyDefs.MyOkCancel.EnableOK = True MyDefs.MyOkCancel.EnableOK = True
End If
End Using End Using
SetCookieText() SetCookieText()
End If End If

2
SCrawler/EncryptCookies.vb
View File

@@ -10,7 +10,7 @@ Imports PersonalUtilities.Tools.Web.Clients
Namespace EncryptCookies Namespace EncryptCookies
Friend Module EncryptFunction Friend Module EncryptFunction
Friend CookiesEncrypted As Boolean = False Friend CookiesEncrypted As Boolean = False
Friend Sub ValidateCookiesEncrypt(ByRef Responser As Response) Friend Sub ValidateCookiesEncrypt(ByRef Responser As Responser)
If Not Responser Is Nothing Then If Not Responser Is Nothing Then
Dim b As Boolean = False Dim b As Boolean = False
With Responser With Responser

4
SCrawler/PluginsEnvironment/Hosts/SettingsHost.vb
View File

@@ -80,7 +80,7 @@ Namespace Plugin.Hosts
Friend ReadOnly Property HasSpecialOptions As Boolean = False Friend ReadOnly Property HasSpecialOptions As Boolean = False
Private ReadOnly _ResponserGetMethod As MethodInfo Private ReadOnly _ResponserGetMethod As MethodInfo
Private ReadOnly _ResponserIsContainer As Boolean = False Private ReadOnly _ResponserIsContainer As Boolean = False
Friend ReadOnly Property Responser As Response Friend ReadOnly Property Responser As Responser
Get Get
If Not _ResponserGetMethod Is Nothing Then If Not _ResponserGetMethod Is Nothing Then
Return _ResponserGetMethod.Invoke(Source, Nothing) Return _ResponserGetMethod.Invoke(Source, Nothing)
@@ -190,7 +190,7 @@ Namespace Plugin.Hosts
If m.MemberType = MemberTypes.Property Then If m.MemberType = MemberTypes.Property Then
PropList.Add(New PropertyValueHost(Source, m)) PropList.Add(New PropertyValueHost(Source, m))
With DirectCast(m, PropertyInfo) With DirectCast(m, PropertyInfo)
If .PropertyType Is GetType(Response) AndAlso m.GetCustomAttribute(Of DoNotUse)() Is Nothing Then _ResponserGetMethod = .GetMethod If .PropertyType Is GetType(Responser) AndAlso m.GetCustomAttribute(Of DoNotUse)() Is Nothing Then _ResponserGetMethod = .GetMethod
End With End With
End If End If
With m.GetCustomAttributes() With m.GetCustomAttributes()