Writing Errors in Multiprocessing Python

The problem is that you're trying to write to the file from many processes in parallel, which isn't synchronized. That means its possible that different processes will try to write at the same time, leading to the oddities you're seeing.

You can solve this by either having a single writer process, with each worker sending the lines to write to that single process, or by synchronizing the writes done by each process using a multiprocessing.Lock.

Using a single writer:

import glob
import multiprocessing
from functools import partial
from threading import Thread

list_files = glob.glob("Protein/*.txt")

def Some_func(out_q, some_file):
    with open(some_file) as some:
        for lines in Some:
            #Do Something
            #edited_lines = func(lines)

            out_q.put(edited_lines)

def write_lines(q):
   with open(file_output) as output:
       for line in iter(q.get, None): # This will end when None is received
           output.write(line)

pool = multiprocessing.Pool(10) # Desired number of threads = 10
m = multiprocessing.Manager()
q = m.Queue()
t = Thread(target=write_lines, args=(q,))
t.start()
pool.map(partial(Some_func, q), list_files)
pool.close()
pool.join()
q.put(None)  # Shut down the writer thread
t.join()

Using a multiprocessing.Lock:

import glob
import multiprocessing
from functools import partial

list_files = glob.glob("Protein/*.txt")

def Some_func(lock, some_file):
    with open(some_file) as some:
        with open(file_output) as output:
            for lines in Some:
                #Do Something
                #edited_lines = func(lines)
                with lock:
                    output.write(edited_lines)


pool = multiprocessing.Pool(10) # Desired number of threads = 10
m = multiprocessing.Manager()
lock = m.Lock()
pool.map(partial(Some_func, lock), list_files)
pool.close()
pool.join()

We need to use a Manager to create the shared objects because you're passing them to a Pool, which requires pickling them. Normal multiprocessing.Lock/multiprocessing.Queue objects can only be passed to a multiprocessing.Process constructor, and will cause an exception when passed to a Pool method like map.