geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-14 15:52:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

2023.12.14.1

Browse Source 
API.Twitter: add additional nodes
This commit is contained in:
Andy
2023-12-14 23:06:55 +03:00
parent 3268bca0d3
commit 5bd79ff3c2
3 changed files with 32 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
Changelog.md
View File

@@ -1,3 +1,10 @@
# 2023.12.14.1
*2023-12-14*
- Fixed
- Twitter: some twitter profiles don't download completely
# 2023.12.14.0 # 2023.12.14.0
*2023-12-14* *2023-12-14*

20
SCrawler/API/Twitter/UserData.vb
View File

@@ -162,7 +162,7 @@ Namespace API.Twitter
Const entry$ = "entry" Const entry$ = "entry"
Dim PostID$ = String.Empty Dim PostID$ = String.Empty
Dim PostDate$, tmpUserId$ Dim PostDate$, tmpUserId$
Dim i% Dim i%, nodeIndx%
Dim dirIndx% = -1 Dim dirIndx% = -1
Dim nodes As List(Of String()) = GetContainerSubnodes() Dim nodes As List(Of String()) = GetContainerSubnodes()
Dim node$() Dim node$()
@@ -170,11 +170,12 @@ Namespace API.Twitter
Dim pinNode As Predicate(Of EContainer) = Function(ee) ee.Value("type").StringToLower = "timelinepinentry" Dim pinNode As Predicate(Of EContainer) = Function(ee) ee.Value("type").StringToLower = "timelinepinentry"
Dim entriesNode As Predicate(Of EContainer) = Function(ee) ee.Name = "entries" Or ee.Name = entry Dim entriesNode As Predicate(Of EContainer) = Function(ee) ee.Name = "entries" Or ee.Name = entry
Dim sourceIdPredicate As Predicate(Of EContainer) = Function(ee) ee.Name = "source_user_id_str" Or ee.Name = "source_user_id" Dim sourceIdPredicate As Predicate(Of EContainer) = Function(ee) ee.Name = "source_user_id_str" Or ee.Name = "source_user_id"
Dim moduleItemsPredicate As Predicate(Of EContainer) = Function(ee) ee.Name.StringToLower = "moduleitems"
Dim newTwitterNodes() As Object = {0, "content", "items"} Dim newTwitterNodes() As Object = {0, "content", "items"}
Dim p As Predicate(Of EContainer) Dim p As Predicate(Of EContainer)
Dim pIndx% Dim pIndx%
Dim isOneNode As Boolean, isPins As Boolean, ExistsDetected As Boolean, userInfoParsed As Boolean = False Dim isOneNode As Boolean, isPins As Boolean, ExistsDetected As Boolean, userInfoParsed As Boolean = False
Dim j As EContainer, rootNode As EContainer, tmpNode As EContainer, nn As EContainer = Nothing Dim j As EContainer, rootNode As EContainer, optionalNode As EContainer, workingNode As EContainer, tmpNode As EContainer, nn As EContainer = Nothing
Dim __parseContainer As Func(Of EContainer, Boolean) = Dim __parseContainer As Func(Of EContainer, Boolean) =
Function(ByVal ee As EContainer) As Boolean Function(ByVal ee As EContainer) As Boolean
@@ -289,18 +290,21 @@ Namespace API.Twitter
End If End If
Else Else
For pIndx = 0 To IIf(dirIndx < 2, 1, 0) For pIndx = 0 To IIf(dirIndx < 2, 1, 0)
optionalNode = Nothing
Select Case dirIndx Select Case dirIndx
Case 0, 1 Case 0, 1
rootNode = j({"data", "user", "result", "timeline_v2", "timeline", "instructions"}) rootNode = j({"data", "user", "result", "timeline_v2", "timeline", "instructions"})
If rootNode.ListExists Then If rootNode.ListExists Then
p = If(pIndx = 0, pinNode, timelineNode) p = If(pIndx = 0, pinNode, timelineNode)
isPins = pIndx = 0 isPins = pIndx = 0
optionalNode = rootNode
rootNode = rootNode.Find(p, False) rootNode = rootNode.Find(p, False)
If rootNode.ListExists Then rootNode = rootNode.Find(entriesNode, False) If rootNode.ListExists Then rootNode = rootNode.Find(entriesNode, False)
End If End If
Case Else Case Else
isPins = False isPins = False
rootNode = j({"globalObjects", "tweets"}) rootNode = j({"globalObjects", "tweets"})
optionalNode = rootNode
End Select End Select
If rootNode.ListExists Then If rootNode.ListExists Then
@@ -311,12 +315,24 @@ Namespace API.Twitter
ProgressPre.Perform() ProgressPre.Perform()
If Not __parseContainer(.Self) Then Exit For If Not __parseContainer(.Self) Then Exit For
Else Else
For nodeIndx = 0 To 1
If nodeIndx = 0 Then
workingNode = rootNode
Else
workingNode = optionalNode
If workingNode.ListExists Then workingNode = workingNode.Find(moduleItemsPredicate, True)
End If
If workingNode.ListExists Then
With workingNode
For Each tmpNode In If(If(.ItemF(newTwitterNodes)?.Count, 0) > 0, For Each tmpNode In If(If(.ItemF(newTwitterNodes)?.Count, 0) > 0,
.ItemF(newTwitterNodes), .ItemF(newTwitterNodes),
.Self) .Self)
ProgressPre.Perform() ProgressPre.Perform()
If Not __parseContainer(tmpNode) Then Exit For If Not __parseContainer(tmpNode) Then Exit For
Next Next
End With
End If
Next
End If End If
End With End With
End If End If

4
SCrawler/My Project/AssemblyInfo.vb
View File

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below: ' by using the '*' as shown below:
' <Assembly: AssemblyVersion("1.0.*")> ' <Assembly: AssemblyVersion("1.0.*")>
<Assembly: AssemblyVersion("2023.12.13.0")> <Assembly: AssemblyVersion("2023.12.14.1")>
<Assembly: AssemblyFileVersion("2023.12.13.0")> <Assembly: AssemblyFileVersion("2023.12.14.1")>
<Assembly: NeutralResourcesLanguage("en")> <Assembly: NeutralResourcesLanguage("en")>