Good day evryone! I am building a database, which is covid-19 related. I have decided there will be many different databases, in which many tiny (growing in the future) collections. Right now I am experimenting and I have a 100 core server, so I’ve decided to multiprocess, however, I always get Autoreconnect error. Before asking the question, I have already serched SO and the Community Forum as well. I have expanded the no of open files:
ulimit -n
1000000
But I still get autoreconnect error. Here is my sample code with sample data:
import pymongo
import mongo_proxy
import time
from sshtunnel import SSHTunnelForwarder
from ssh_pymongo import MongoSession
from multiprocessing import *
#Number of cores
x = cpu_count()
print(x)
myClient = pymongo.MongoClient("mongodb://127.0.0.1:27017/")
def listFile(fileName):
fOpen = open(fileName)
listFile = fOpen.readlines()
arrOfArrs = []
tmp1 = []
for i in listFile:
# print(i)
if i.startswith(">"):
if len(tmp1) > 1:
arrOfArrs.append(tmp1)
tmp1 = []
tmp1.append(i.strip())
else:
tmp1.append(i.strip())
#print(listFile)
return arrOfArrs
def createCollectionPDB(fP):
lineName = “”
lineFASTA = “”
colName = “”
PDBName = “”
chainIDName = “”
typeOfMol = “”
molLen = “”
proteinName = “”
for i in fP:
print(“test”, i)
print(lineName)
if “>” in i:
lineName = i.strip()
print(“LINE NAME”)
colName = lineName.split(" “)[0].strip()[1:]
print(“COLNAME”, colName)
PDBName = lineName.split(”")[0].strip()
chainIDName = colName.split("")[-1].strip()
typeOfMol = lineName.split(" “)[1].strip().split(”:")[1].strip()
molLen = lineName.split(" “)[2].strip().split(”:")[-1].strip()#[3].split(" “)[0].strip()
proteinName = lineName.split(” ")[-1].strip()
print(colName, PDBName, chainIDName, typeOfMol, molLen, proteinName)
else:
print(“ELSE”, colName)
lineFASTA = i.strip()
dict2Write={“PDB_ID” : PDBName, “Chain_ID” : chainIDName, “Molecule Type” : typeOfMol, “Length” : molLen, “Protein_Name” : proteinName, “FASTA” : lineFASTA}
myClient = pymongo.MongoClient(“mongodb://127.0.0.1:27017/”)
myNewDB = myClient[“MyPrj_PDBs”]
newCol = myNewDB[colName]
x = newCol.insert_one(dict2Write)
print(“PDB”, x.inserted_id)#’’’
myClient.close()
myPool = Pool(processes=x-2)
myPool.map(createCollectionPDB, listFile(“datum/pdb_seqres.txt”))
myPool.join()
myPool.close()