Merge remote-tracking branch 'origin/main'

Author: NishilAmin1213

resources/repositories/c#_repos.txt

@@ -1,14 +1,13 @@
-arrow
+activemq-nms-amqp
+activemq-nms-api
+activemq-nms-ems
+activemq-nms-msmq
+activemq-nms-openwire
+activemq-nms-stomp
+activemq-nms-xms
+logging-log4net
 lucenenet
-log4net
-qpid
-thrift
-ignite
-cordova
-etch
-chemistry
-avro
-hadoop
-hbase
-hive
-kafka
+pulsar-dotpulsar
+qpid-proton-dotnet
+rocketmq-client-csharp
+usergrid-dotnet

resources/repositories/c++_repos.txt

@@ -0,0 +1,10 @@
+brpc
+datasketches-cpp
+incubator-pegasus
+kudu
+kvrocks
+mesos
+nifi-minifi-cpp
+pulsar-client-cpp
+singa
+trafficserver

resources/repositories/java_repos.txt

@@ -10,41 +10,31 @@ activemq
 activemq-activeio
 activemq-artemis
 activemq-cli-tools
-activemq-nms-openwire-generator
 activemq-openwire
 airavata
-allura
 ambari
 ant
-ant-antlibs-compress
 ant-antlibs-dotnet
-ant-antlibs-props
 ant-antlibs-vss
 ant-easyant-tasks
 ant-ivy
 ant-ivyde
 apex-core
-archiva
 archiva-redback-components
 archiva-redback-core
 archiva-sandbox
 aries
-aries-blueprint
 aries-cdi
 aries-containers
 aries-jax-rs-whiteboard
 aries-jpa
 aries-rsa
-aries-spifly
 aries-tx-control
-aries-util
 arrow
-asterixdb
 asterixdb-bad
 asterixdb-hyracks
 atlas
 avro
-axis2-java
 bigtop
 bigtop-manager
 bookkeeper
@@ -68,7 +58,6 @@ commons-chain
 commons-cli
 commons-codec
 commons-collections
-commons-collections4
 commons-compress
 commons-configuration
 commons-csv
@@ -100,13 +89,11 @@ commons-validator
 commons-vfs
 commons-weaver
 continuum
-cordova-plugin-compat
 creadur-rat
 crunch
 ctakes
 curator
 cxf
-cxf-build-utils
 cxf-dosgi
 cxf-fediz
 cxf-xjc-utils
@@ -116,7 +103,6 @@ datasketches-memory
 datasketches-pig
 datasketches-vector
 db-jdo
-db-torque
 ddlutils
 deltaspike
 directmemory-lightning
@@ -127,17 +113,12 @@ directory-ldap-api
 directory-mavibot
 directory-samples
 directory-server
-directory-sms
 directory-studio
-drill
-druid
-dubbo
 dubbo-rpc-jsonrpc
 empire-db
 felix
 fineract
 flex-blazeds
-flink
 flume
 fluo
 fluo-recipes
@@ -159,7 +140,6 @@ gobblin
 gora
 groovy
 guacamole-client
-hadoop
 hawq
 hbase
 hbase-operator-tools
@@ -174,9 +154,6 @@ hudi
 ignite
 incubator-batchee
 incubator-hivemall
-incubator-kie-benchmarks
-incubator-kie-drools
-incubator-kie-optaplanner
 incubator-optiq-linq4j
 incubator-retired-gossip
 incubator-retired-hdt
@@ -204,11 +181,9 @@ incubator-tuweni
 incubator-twill
 isis
 jackrabbit
-jackrabbit-filevault
 jackrabbit-filevault-package-maven-plugin
 jackrabbit-oak
 jackrabbit-ocm
-james
 james-jdkim
 james-jsieve
 james-jspf
@@ -222,11 +197,9 @@ jspwiki
 kafka
 kalumet
 kandula
-karaf
 karaf-boot
 karaf-cellar
 karaf-decanter
-kylin
 lens
 logging-flume
 logging-log4j-boot
@@ -237,7 +210,6 @@ manifoldcf
 manifoldcf-integration-solr-3.x
 manifoldcf-integration-solr-4.x
 marmotta
-maven
 maven-2
 maven-ant-tasks
 maven-app-engine
@@ -258,11 +230,8 @@ metamodel
 metamodel-membrane
 mina
 mina-sshd
-myfaces-extcdi
-myfaces-extval
 myfaces-scripting
 netbeans-html4j
-nifi
 nifi-maven
 nifi-minifi
 nutch
@@ -313,53 +282,42 @@ sandesha
 sanselan
 santuario-java
 savan
-seatunnel
 sentry
 servicecomb-java-chassis
 servicemix
 servicemix-archetypes
-servicemix-bundles
 servicemix-components
 servicemix-maven-plugins
 servicemix-specs
 servicemix-utils
 servicemix3
-servicemix4-bundles
 servicemix4-features
 servicemix4-kernel
 servicemix4-nmr
 servicemix4-specs
 shardingsphere-elasticjob
-shardingsphere-example
 shiro
 sis
-skywalking
-sling
 sling-org-apache-sling-adapter
 sling-org-apache-sling-api
 sling-org-apache-sling-auth-core
 sling-org-apache-sling-auth-form
 sling-org-apache-sling-bundleresource-impl
-sling-org-apache-sling-caconfig-api
 sling-org-apache-sling-caconfig-impl
 sling-org-apache-sling-caconfig-integration-tests
 sling-org-apache-sling-caconfig-spi
 sling-org-apache-sling-commons-classloader
 sling-org-apache-sling-commons-compiler
 sling-org-apache-sling-commons-contentdetection
-sling-org-apache-sling-commons-johnzon
 sling-org-apache-sling-commons-log
 sling-org-apache-sling-commons-logservice
-sling-org-apache-sling-commons-messaging
 sling-org-apache-sling-commons-messaging-mail
 sling-org-apache-sling-commons-metrics
 sling-org-apache-sling-commons-metrics-rrd4j
 sling-org-apache-sling-commons-mime
 sling-org-apache-sling-commons-osgi
 sling-org-apache-sling-commons-scheduler
-sling-org-apache-sling-commons-threaddump
 sling-org-apache-sling-commons-threads
-sling-org-apache-sling-discovery-api
 sling-org-apache-sling-discovery-base
 sling-org-apache-sling-discovery-commons
 sling-org-apache-sling-discovery-impl
@@ -371,33 +329,26 @@ sling-org-apache-sling-event-dea
 solr
 spark
 sqoop
-stanbol
 storm
 struts
-struts-annotations
-struts-extras
 struts-sandbox
 struts1
 subversion
 synapse
 syncope
-tapestry
 tez
-tika
 tiles-autotag
 tiles-request
 tomcat
 tomcat-maven-plugin
 tomcat55
 tomcat80
 tomee
-turbine
 tuscany-das
 tuscany-sca-1.x
 tuscany-sca-2.x
 tuscany-sdo
 twill
-uima
 uima-addons
 uima-sandbox
 uima-uimafit

resources/repositories/kotlin_repos.txt

@@ -1,15 +1,2 @@
-spark
-beam
-kafka
-flink
-druid
-pulsar
-nifi
-cassandra
-hadoop
-hbase
-hive
-storm
-superset
-airflow
-mahout
+logging-log4j-kotlin
+incubator-retired-amaterasu

resources/repositories/python_repos.txt

@@ -1,20 +1,9 @@
 airflow
-beam
-libcloud
-mxnet
-spamassassin
-superset
-zookeeper
-hadoop
-hbase
-hive
-kafka
-spark
-storm
-flink
-druid
-superset
-pulsar
-nifi
-cassandra
-mahout
+allura
+openserverless-operator
+iceberg-python
+airflow-client-python
+skywalking-python
+ignite-python-thin-client
+pulsar-client-python
+hunter

resources/repositories/rust_repos.txt

@@ -31,12 +31,12 @@ def read_repositories(language: str) -> List[Repository]:
 
         return [apache_repo_from_name(repo_name) for repo_name in repositories]
 
-def read_commits(repository_url: str, file_extension: str, final_date: Optional[datetime] = None) -> Generator[Commit, None, None]:
+def read_commits(repository_url: str, file_extensions: List[str], final_date: Optional[datetime] = None) -> Generator[Commit, None, None]:
     """
     Reads commits from a repository using PyDriller.
 
     @param: repository_url: The URL of the repository.
-    @param: file_extension: The file extension to search for.
+    @param: file_extensions: The file extensions to search for.
     @param: final_date: Date to read commits up until from the given repository.
 
     @return: A generator of Commit objects.
@@ -45,7 +45,7 @@ def read_commits(repository_url: str, file_extension: str, final_date: Optional[
         raise ValueError("Final date must be in the past.")
     to_date = final_date if final_date else datetime.now()
 
-    return DrillerRepo(repository_url, only_modifications_with_file_types=[file_extension], to=to_date).traverse_commits()
+    return DrillerRepo(repository_url, only_modifications_with_file_types=file_extensions, to=to_date).traverse_commits()
 
 def repo_from_url(repo_url: str):
     repo_name = re.search(r"github\.com/[^/]+/([^/.]+)", repo_url).group(1) if re.search(r"github\.com/[^/]+/([^/.]+)", repo_url) else None

src/infrastructure/repository_utils.py

@@ -1,6 +1,6 @@
 from collections import defaultdict
 from src.mining import commit_retrieval as retrieval
-from src.models.LanguageFileHandler import LanguageFileHandler
+from src.models.file_handlers.LanguageFileHandler import LanguageFileHandler
 
 def gather_commits_and_tests(repo_name, file_handler: LanguageFileHandler):
     """
@@ -82,4 +82,4 @@ def calculate_average_commit_size(commits, test_files):
             counter += 1
             complete_indexes.append(test_file[0])
 
-    return round(total/counter, 1)
+    return round(total/counter, 1) if counter != 0 else 0

src/mining/commit_processing.py

@@ -4,15 +4,15 @@
 from src.infrastructure import repository_utils as repository_utils
 from src.infrastructure import file_utils as file_utils
 from src.infrastructure import serialize as serializer
-from src.models.LanguageFileHandler import LanguageFileHandler
+from src.models.file_handlers.LanguageFileHandler import LanguageFileHandler
 from src.models.CustomCommit import CustomCommit
 from src.models.Repository import Repository
 
 def _retrieve_files(modified_files, file_handler: LanguageFileHandler):
     files = []
 
     for file in modified_files:
-        if file_handler.file_extension in file.filename:
+        if any(file_extension in file.filename for file_extension in file_handler.file_extensions):
             files.append(file.filename)
 
     return files
@@ -21,7 +21,7 @@ def _retrieve_commits(repo_url, file_handler: LanguageFileHandler, final_date =
     commits = []
 
     try:
-        for commit in repository_utils.read_commits(repo_url, file_handler.file_extension, final_date):
+        for commit in repository_utils.read_commits(repo_url, file_handler.file_extensions, final_date):
             files = _retrieve_files(commit.modified_files, file_handler)
             commits.append(CustomCommit(commit.hash, files, commit.author, commit.author_date))
     except Exception as e: