I'm doing some file manipulation work with node.js and many of the packages that I use require a "path" to be sent so that they can open the file, do some work, and so on.
But I'm parsing millions of files and rather than actually store them on disk, I'd like to store them in memory. The contents of the files are all in my database and I'd hate to write them to disk, just to do my insanely awesome work on them.
So is such a thing possible?
It looks like it's possible look at this article how to do it creating writable memory stream
Maybe create a Buffer using let myBuffer = Buffer.from(fetchedData) after fetching each file from the database then using myBuffer.toString() to change the file to a string and parse/manipulate it however you like
Because I have not found a single library that suited my needs, I have created diskette. It is a tiny library that tries to efficiently store data such as buffers and strings and make it streamable. I use it in my own projects and so far it works like a charm.
Please let me know if you found a bug or if there's anything more that can be added to it.
If you are using a Linux system then you can merely create your own tmpfs (RAM) via the following nodejs code. This creates a tmpfs at /mnt/myramdisk as per the node code. The directory /mnt/myramdisk must be manually created beforehand obviously via mkdir /mnt/myramdisk.
var MountPoint='/mnt/myramdisk';
var TextFile='/MyTextFileInRAM.txt';
var RAM_Size='512m';
const fs = require('fs');
const { exec } = require('child_process');
exec("awk '{print $2}' /proc/mounts | grep "+MountPoint, (err, stdout, stderr) => {
if (err) {
// node couldn't execute the command
//console.log(err);
console.log(MountPoint+' is Not Mounted yet. Im mounting it now:\n');
NotMountedYetSoMountIt();
return;
}
// the *entire* stdout and stderr (buffered)
if(stdout)
{
console.log(MountPoint+' is Already Mounted');
TextToWriteToFileOnTMPFS();
}
});
function NotMountedYetSoMountIt()
{
const { exec } = require('child_process');
exec('df -h && echo && mount -t tmpfs -o size='+RAM_Size+' tmpfs '+MountPoint+' && echo && df -h', (err, stdout, stderr) => {
if (err) {
// node couldn't execute the command
return;
}
// the *entire* stdout and stderr (buffered)
console.log(`stdout: ${stdout}`);
TextToWriteToFileOnTMPFS();
console.log(`stderr: ${stderr}`);
});
}
function TextToWriteToFileOnTMPFS()
{
let TextToWrite = 'Hello\n' +
'world @'+CurrentTime();
fs.writeFile(MountPoint+TextFile, TextToWrite, (err) => {
// throws an error, you could also catch it here
if (err) throw err;
// success case, the file was saved
console.log('saved!');
});
}
function addZero(i) {
if (i < 10) {
i = "0" + i;
}
return i;
}
function CurrentTime()
{
var d = new Date();
var h = addZero(d.getHours());
var m = addZero(d.getMinutes());
var s = addZero(d.getSeconds());
return h + ":" + m + ":" + s;
}
Output:
root@box:/daemons#node tmpfs_test.js && cat /mnt/myramdisk/MyTextFileInRAM.txt
/mnt/myramdisk is Not Mounted yet. Im mounting it now:
stdout: Filesystem Size Used Avail Use% Mounted on
udev 7.8G 0 7.8G 0% /dev
tmpfs 1.6G 1.4M 1.6G 1% /run
/dev/sda2 938G 436G 454G 49% /
tmpfs 7.8G 449M 7.4G 6% /dev/shm
tmpfs 5.0M 0 5.0M 0% /run/lock
tmpfs 7.8G 0 7.8G 0% /sys/fs/cgroup
/dev/loop1 90M 90M 0 100% /snap/core/8213
/dev/sda1 511M 6.1M 505M 2% /boot/efi
/dev/loop2 90M 90M 0 100% /snap/core/8268
Filesystem Size Used Avail Use% Mounted on
udev 7.8G 0 7.8G 0% /dev
tmpfs 1.6G 1.4M 1.6G 1% /run
/dev/sda2 938G 436G 454G 49% /
tmpfs 7.8G 449M 7.4G 6% /dev/shm
tmpfs 5.0M 0 5.0M 0% /run/lock
tmpfs 7.8G 0 7.8G 0% /sys/fs/cgroup
/dev/loop1 90M 90M 0 100% /snap/core/8213
/dev/sda1 511M 6.1M 505M 2% /boot/efi
/dev/loop2 90M 90M 0 100% /snap/core/8268
tmpfs 512M 0 512M 0% /mnt/myramdisk
stderr:
saved!
Hello
world @23:09:15
root@box:/daemons# node tmpfs_test.js && cat /mnt/myramdisk/MyTextFileInRAM.txt
/mnt/myramdisk is Already Mounted
saved!
Hello
world @23:09:19
fsmodule is involved, which is likely given the requirement of apath, then they need to be files in an accessible file system. You should be able to use a virtual disk or in-memory fs, such as tmpfs. But, unless the database can pretend to be a file system itself, you'll have to write them out. Or, find alternate methods or packages that work with Streams or Buffers rather than paths.