geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-15 08:12:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

3.0.0.0

Browse Source 
Parsing profiles descriptions (Reddit and Twitter) and updating it
Filters: deleted, suspended, dates
Collections containing deleted profiles are marked in blue
Marked collection context elements
Find profile in the main window from the info form
New hotkeys in the info form: up, down, find, enter
New hotkey in the main window: enter
New list refill algo
Added copying user pictures from all channels
Changed view modes
Changed comparer and ToString of UserDataBase
New parameter added to channels stats (my users)
Added view mode "details"
Fixed twitter files overriding
Fixed full parsing of reddit posts
Fixed Insta timers and minors
Fixed library fatal
Removed UserDataBind comparer override
Added GetUserMediaOnly for reddit users from channels
Added Reddit availability check with DownDetector
Added PLUGINS
This commit is contained in:
Andy
2022-03-17 21:15:22 +03:00
parent 19373ec4ba
commit 05c84c2c08
135 changed files with 7889 additions and 3794 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
SCrawler/API/Twitter/Declarations.vb
View File

@@ -10,6 +10,7 @@ Imports PersonalUtilities.Functions.XML.Base
Imports PersonalUtilities.Functions.RegularExpressions
Namespace API.Twitter
Friend Module Declarations
Friend Const TwitterSite As String = "Twitter"
Friend DateProvider As New ADateTime(ADateTime.Formats.BaseDateTime)
Friend ReadOnly VideoNode As NodeParams() = {New NodeParams("video_info", True, True, True, True, 10)}
Friend ReadOnly VideoSizeRegEx As RParams = RParams.DMS("\d+x(\d+)", 1, EDP.ReturnValue)

99
SCrawler/API/Twitter/SiteSettings.vb Normal file
View File

@@ -0,0 +1,99 @@
' Copyright (C) 2022 Andy
' This program is free software: you can redistribute it and/or modify
' it under the terms of the GNU General Public License as published by
' the Free Software Foundation, either version 3 of the License, or
' (at your option) any later version.
'
' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY
Imports SCrawler.Plugin
Imports SCrawler.Plugin.Attributes
Imports PersonalUtilities.Tools
Imports PersonalUtilities.Functions.RegularExpressions
Imports SCrawler.API.Base
Namespace API.Twitter
<Manifest("AndyProgram_Twitter"), UseClassAsIs>
Friend Class SiteSettings : Inherits SiteSettingsBase
Friend Const Header_Authorization As String = "authorization"
Friend Const Header_Token As String = "x-csrf-token"
Friend Overrides ReadOnly Property Icon As Icon
Get
Return My.Resources.TwitterIcon
End Get
End Property
Friend Overrides ReadOnly Property Image As Image
Get
Return My.Resources.TwitterPic400
End Get
End Property
<PropertyOption(AllowNull:=False, ControlText:="Authorization",
ControlToolTip:="Set authorization from [authorization] response header. This field must start from [Bearer] key word")>
Private ReadOnly Property Auth As PropertyValue
<PropertyOption(AllowNull:=False, ControlText:="Token", ControlToolTip:="Set token from [x-csrf-token] response header")>
Private ReadOnly Property Token As PropertyValue
Friend Overrides ReadOnly Property Responser As WEB.Response
Friend Sub New()
MyBase.New(TwitterSite)
Responser = New WEB.Response($"{SettingsFolderName}\Responser_{Site}.xml")
Dim a$ = String.Empty
Dim t$ = String.Empty
With Responser
If .File.Exists Then
.LoadSettings()
With .Headers
If .ContainsKey(Header_Authorization) Then a = .Item(Header_Authorization)
If .ContainsKey(Header_Token) Then t = .Item(Header_Token)
End With
Else
.ContentType = "application/json"
.Accept = "*/*"
.CookiesDomain = "twitter.com"
.Decoders.Add(SymbolsConverter.Converters.Unicode)
With .Headers
.Add("sec-ch-ua", " Not;A Brand" & Chr(34) & ";v=" & Chr(34) & "99" & Chr(34) & ", " & Chr(34) &
"Google Chrome" & Chr(34) & ";v=" & Chr(34) & "91" & Chr(34) & ", " & Chr(34) & "Chromium" &
Chr(34) & ";v=" & Chr(34) & "91" & Chr(34))
.Add("sec-ch-ua-mobile", "?0")
.Add("sec-fetch-dest", "empty")
.Add("sec-fetch-mode", "cors")
.Add("sec-fetch-site", "same-origin")
.Add(Header_Token, String.Empty)
.Add("x-twitter-active-user", "yes")
.Add("x-twitter-auth-type", "OAuth2Session")
.Add(Header_Authorization, String.Empty)
End With
.SaveSettings()
End If
End With
Auth = New PropertyValue(a, GetType(String), Sub(v) ChangeResponserFields(NameOf(Auth), v))
Token = New PropertyValue(t, GetType(String), Sub(v) ChangeResponserFields(NameOf(Token), v))
UserRegex = RParams.DMS("[htps:/]{7,8}.*?twitter.com/([^/]+)", 1)
UrlPatternUser = "https://twitter.com/{0}"
ImageVideoContains = "twitter"
End Sub
Private Sub ChangeResponserFields(ByVal PropName As String, ByVal Value As Object)
If Not PropName.IsEmptyString Then
Dim f$ = String.Empty
Select Case PropName
Case NameOf(Auth) : f = Header_Authorization
Case NameOf(Token) : f = Header_Token
End Select
If Not f.IsEmptyString Then
If Responser.Headers.Count > 0 AndAlso Responser.Headers.ContainsKey(f) Then Responser.Headers.Remove(f)
If Not CStr(Value).IsEmptyString Then Responser.Headers.Add(f, CStr(Value))
Responser.SaveSettings()
End If
End If
End Sub
Friend Overrides Function GetInstance(ByVal What As ISiteSettings.Download) As IPluginContentProvider
Return New UserData
End Function
Friend Overrides Function GetSpecialDataF(ByVal URL As String) As IEnumerable(Of UserMedia)
Return UserData.GetVideoInfo(URL, Responser)
End Function
End Class
End Namespace

47
SCrawler/API/Twitter/UserData.vb
View File

@@ -16,27 +16,25 @@ Imports SCrawler.API.Base
Namespace API.Twitter
Friend Class UserData : Inherits UserDataBase
#Region "Declarations"
Friend Overrides Property Site As Sites = Sites.Twitter
Private ReadOnly _DataNames As List(Of String)
#End Region
#Region "Initializer"
Friend Sub New(ByVal u As UserInfo, Optional ByVal _LoadUserInformation As Boolean = True)
User = u
If _LoadUserInformation Then LoadUserInformation()
Friend Sub New()
_DataNames = New List(Of String)
End Sub
#End Region
#Region "Load and Update user info"
Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean)
End Sub
#End Region
#Region "Download functions"
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
If _ContentList.Count > 0 Then _DataNames.ListAddList(_ContentList.Select(Function(c) c.File.File), LAP.ClearBeforeAdd, LAP.NotContainsOnly)
DownloadData(String.Empty, Token)
End Sub
Private Overloads Sub DownloadData(ByVal POST As String, ByVal Token As CancellationToken)
Dim URL$ = String.Empty
Try
Dim PostID$ = String.Empty
Dim PostDate$
Dim PostDate$, dName$
Dim m As EContainer, nn As EContainer, s As EContainer
Dim NewPostDetected As Boolean = False
Dim ExistsDetected As Boolean = False
@@ -61,6 +59,8 @@ Namespace API.Twitter
If Not ID.IsEmptyString Then UpdateUserInformation()
End If
If UserDescriptionNeedToUpdate() AndAlso nn.Value({"user"}, "screen_name") = Name Then UserDescriptionUpdate(nn.Value({"user"}, "description"))
'Date Pattern:
'Sat Jan 01 01:10:15 +0000 2000
If nn.Contains("created_at") Then PostDate = nn("created_at").Value Else PostDate = String.Empty
@@ -81,8 +81,12 @@ Namespace API.Twitter
If Not s Is Nothing AndAlso s.Count > 0 Then
For Each m In s
If m.Count > 0 AndAlso m.Contains("media_url") Then
_TempMediaList.ListAddValue(MediaFromData(m("media_url").Value,
PostID, PostDate, GetPictureOption(m)), LNC)
dName = UrlFile(m("media_url").Value)
If Not dName.IsEmptyString AndAlso Not _DataNames.Contains(dName) Then
_DataNames.Add(dName)
_TempMediaList.ListAddValue(MediaFromData(m("media_url").Value,
PostID, PostDate, GetPictureOption(m)), LNC)
End If
End If
Next
End If
@@ -99,12 +103,12 @@ Namespace API.Twitter
ProcessException(ex, Token, $"data downloading error [{URL}]")
End Try
End Sub
Friend Shared Function GetVideoInfo(ByVal URL As String) As IEnumerable(Of UserMedia)
Friend Shared Function GetVideoInfo(ByVal URL As String, ByVal resp As Response) As IEnumerable(Of UserMedia)
Try
If URL.Contains("twitter") Then
Dim PostID$ = RegexReplace(URL, RParams.DM("(?<=/)\d+", 0))
If Not PostID.IsEmptyString Then
Dim r$ = DirectCast(Settings(Sites.Twitter).Responser.Copy(), Response).
Dim r$ = DirectCast(resp.Copy(), Response).
GetResponse($"https://api.twitter.com/1.1/statuses/show.json?id={PostID}",, EDP.ReturnValue)
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r)
@@ -147,7 +151,14 @@ Namespace API.Twitter
Private Function CheckVideoNode(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String) As Boolean
Try
Dim URL$ = GetVideoNodeURL(w)
If Not URL.IsEmptyString Then _TempMediaList.ListAddValue(MediaFromData(URL, PostID, PostDate), LNC) : Return True
If Not URL.IsEmptyString Then
Dim f$ = UrlFile(URL)
If Not f.IsEmptyString AndAlso Not _DataNames.Contains(f) Then
_DataNames.Add(f)
_TempMediaList.ListAddValue(MediaFromData(URL, PostID, PostDate), LNC)
End If
Return True
End If
Return False
Catch ex As Exception
LogError(ex, "[API.Twitter.UserData.CheckVideoNode]")
@@ -177,6 +188,14 @@ Namespace API.Twitter
End Function
Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken)
End Sub
Private Function UrlFile(ByVal URL As String) As String
Try
Dim f As SFile = CStr(RegexReplace(LinkFormatterSecure(RegexReplace(URL.Replace("\", String.Empty), LinkPattern)), FilesPattern))
If Not f.IsEmptyString Then Return f.File Else Return String.Empty
Catch ex As Exception
Return String.Empty
End Try
End Function
#End Region
Private Shared Function MediaFromData(ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String,
Optional ByVal _PictureOption As String = "") As UserMedia
@@ -206,5 +225,9 @@ Namespace API.Twitter
End If
Return 1
End Function
Protected Overrides Sub Dispose(ByVal disposing As Boolean)
If Not disposedValue And disposing Then _DataNames.Clear()
MyBase.Dispose(disposing)
End Sub
End Class
End Namespace