Get all unique values in a JavaScript array (remove duplicates) [duplicate]

You don't need .indexOf() at all; you can do this O(n):

function SelectDistinct(array) {
    const seenIt = new Set();

    return array.filter(function (val) {
        if (seenIt.has(val)) { 
            return false;
        }

        seenIt.add(val);

        return true;
    });
}

var hasDuplicates = [1,2,3,4,5,5,6,7,7];
console.log(SelectDistinct(hasDuplicates)) //[1,2,3,4,5,6,7]

If you don't want to use .filter():

function SelectDistinct(array) {
    const seenIt = new Set();
    const distinct = [];

    for (let i = 0; i < array.length; i++) {
        const value = array[i];

        if (!seenIt.has(value)) {
            seenIt.add(value);
            distinct.push(value);
        }
    }
    
    return distinct; 
    /* you could also drop the 'distinct' array and return 'Array.from(seenIt)', which converts the set object to an array */
}

If you want to remove duplicates, return the whole objects and want to use ES6 Set and Map syntax, and also run only one loop, you can try this, to get unique ids:

const collection = [{id:3, name: "A"}, {id:3, name: "B"}, {id:4, name: "C"}, {id:5, name: "D"}]

function returnUnique(itemsCollection){
  const itemsMap = new Map();

  itemsCollection.forEach(item => {
    if(itemsMap.size === 0){
      itemsMap.set(item.id, item)       
    }else if(!itemsMap.has(item.id)){
      itemsMap.set(item.id, item)
    }
  });
  
    return [...new Set(itemsMap.values())];
 }

console.log(returnUnique(collection));

For an array of tuples, I'll throw things into a Map and let it do the work. With this approach, you have to be mindful about the key you want to be using:

const arrayOfArraysWithDuplicates = [
    [1, 'AB'],
    [2, 'CD'],
    [3, 'EF'],
    [1, 'AB'],
    [2, 'CD'],
    [3, 'EF'],
    [3, 'GH'],
]

const uniqueByFirstValue = new Map();
const uniqueBySecondValue = new Map();

arrayOfArraysWithDuplicates.forEach((item) => {
    uniqueByFirstValue.set(item[0], item[1]);
    uniqueBySecondValue.set(item[1], item[0]);
});

let uniqueList = Array.from( uniqueByFirstValue, ( [ value, name ] ) => ( [value, name] ) );

console.log('Unique by first value:');
console.log(uniqueList);

uniqueList = Array.from( uniqueBySecondValue, ( [ value, name ] ) => ( [value, name] ) );

console.log('Unique by second value:');
console.log(uniqueList);

Output:

Unique by first value:
[ [ 1, 'AB' ], [ 2, 'CD' ], [ 3, 'GH' ] ]

Unique by second value:
[ [ 'AB', 1 ], [ 'CD', 2 ], [ 'EF', 3 ], [ 'GH', 3 ] ]

If order is not important then we can make an hash and get the keys to make unique array.

var ar = [1,3,4,5,5,6,5,6,2,1];
var uarEle = {};
links.forEach(function(a){ uarEle[a] = 1; });
var uar = keys(uarEle)

uar will be having the unique array elements.

I looked at Joeytje50's code on jsperf who has compared a number of alternatives. His code had many minor typos, which made a difference in the performance and the correctness.

More importantly, he is testing on a very small array. I made an array with 1000 integers. Each integer was 100 times a random integer between 0 and 1000. This makes for about 1000/e = 368 duplicates on the average. The results are at jsperf.

This is a much more realistic scenario of where efficiency might be needed. These changes make dramatic changes in the claims (specifically the code touted as fastest is nowhere near fast). The obvious winners are where hashing techniques are used, with the best one being

Array.prototype.getUnique3 = function(){
   var u = Object.create(null), a = [];
   for(var i = 0, l = this.length; i < l; ++i){
      if(this[i] in u) continue;
      a.push(this[i]);
      u[this[i]] = 1;
   }
   return a.length;
}

If you have an array of objects, and you want a uniqueBy function, say, by an id field:

function uniqueBy(field, arr) {
   return arr.reduce((acc, curr) => {
     const exists = acc.find(v => v[field] === curr[field]);
     return exists ? acc : acc.concat(curr);
   }, [])
}

I know this has been answered to death already... but... no one has mentioned the javascript implementation of linq. Then the .distinct() method can be used - and it makes the code super easy to read.

var Linq = require('linq-es2015');
var distinctValues =  Linq.asEnumerable(testValues)
            .Select(x)
            .distinct()
            .toArray();

var testValues = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 1];

var distinctValues = Enumerable.asEnumerable(testValues)
  .distinct()
  .toArray();

console.log(distinctValues);

<script src="https://npmcdn.com/linq-es5/dist/linq.js"></script>

This is an ES6 function which removes duplicates from an array of objects, filtering by the specified object property

function dedupe(arr = [], fnCheck = _ => _) {
  const set = new Set();
  let len = arr.length;

  for (let i = 0; i < len; i++) {
    const primitive = fnCheck(arr[i]);
    if (set.has(primitive)) {
      // duplicate, cut it
      arr.splice(i, 1);
      i--;
      len--;
    } else {
      // new item, add it
      set.add(primitive);
    }
  }

  return arr;
}

const test = [
    {video:{slug: "a"}},
    {video:{slug: "a"}},
    {video:{slug: "b"}},
    {video:{slug: "c"}},
    {video:{slug: "c"}}
]
console.log(dedupe(test, x => x.video.slug));

// [{video:{slug: "a"}}, {video:{slug: "b"}}, {video:{slug: "c"}}]

The Object answer above does not seem to work for me in my use case with Objects.

I have modified it as follows:

var j = {};

this.forEach( function(v) {
   var typ = typeof v;
   var v = (typ === 'object') ? JSON.stringify(v) : v;

   j[v + '::' + typ] = v;
});

return Object.keys(j).map(function(v){
  if ( v.indexOf('::object') > -1 ) {
    return JSON.parse(j[v]);
  }

  return j[v];
});

This seems to now work correctly for objects, arrays, arrays with mixed values, booleans, etc.

To filter-out undefined and null values because most of the time you do not need them.

const uniques = myArray.filter(e => e).filter((e, i, a) => a.indexOf(e) === i);

or

const uniques = [...new Set(myArray.filter(e => e))];

Sometimes I need to get unique occurrences from an array of objects. Lodash seems like a nice helper but I don't think filtering an array justifies adding a dependency to a project.

Let's assume the comparison of two objects poses on comparing a property, an id for example.

const a = [{id: 3}, {id: 4}, {id: 3}, {id: 5}, {id: 5}, {id: 5}];

Since we all love one line snippets, here is how it can be done:

a.reduce((acc, curr) => acc.find(e => e.id === curr.id) ? acc : [...acc, curr], [])

I have a solution that uses es6 reduce and find array helper methods to remove duplicates.

let numbers = [2, 2, 3, 3, 5, 6, 6];

const removeDups = array => {
  return array.reduce((acc, inc) => {
    if (!acc.find(i => i === inc)) {
      acc.push(inc);
    }
    return acc;
  }, []);
}

console.log(removeDups(numbers)); /// [2,3,5,6]

Making an array of unique arrays, using field[2] as an Id:

const arr = [
  ['497', 'Q0', 'WTX091-B06-138', '0', '1.000000000', 'GROUP001'],
  ['497', 'Q0', 'WTX091-B09-92', '1', '0.866899288', 'GROUP001'],
  ['497', 'Q0', 'WTX091-B09-92', '2', '0.846036819', 'GROUP001'],
  ['497', 'Q0', 'WTX091-B09-57', '3', '0.835025326', 'GROUP001'],
  ['497', 'Q0', 'WTX091-B43-79', '4', '0.765068215', 'GROUP001'],
  ['497', 'Q0', 'WTX091-B43-56', '5', '0.764211464', 'GROUP001'],
  ['497', 'Q0', 'WTX091-B44-448', '6', '0.761701704', 'GROUP001'],
  ['497', 'Q0', 'WTX091-B44-12', '7', '0.761701704', 'GROUP001'],
  ['497', 'Q0', 'WTX091-B49-128', '8', '0.747434800', 'GROUP001'],
  ['497', 'Q0', 'WTX091-B18-17', '9', '0.746724770', 'GROUP001'],
  ['497', 'Q0', 'WTX091-B19-374', '10', '0.733379549', 'GROUP001'],
  ['497', 'Q0', 'WTX091-B19-344', '11', '0.731421782', 'GROUP001'],
  ['497', 'Q0', 'WTX091-B09-92', '12', '0.726450470', 'GROUP001'],
  ['497', 'Q0', 'WTX091-B19-174', '13', '0.712757036', 'GROUP001']
];


arr.filter((val1, idx1, arr) => !!~val1.indexOf(val1[2]) &&
  !(arr.filter((val2, idx2) => !!~val2.indexOf(val1[2]) &&
    idx2 < idx1).length));

console.log(arr);

This solution should be very fast, and will work in many cases.

1. Convert the indexed array items to object keys

2. Use Object.keys function

   var indexArray = ["hi","welcome","welcome",1,-9];
   var keyArray = {};
   indexArray.forEach(function(item){ keyArray[item]=null; });
   var uniqueArray = Object.keys(keyArray);
   

I have a simple example where we can remove objects from array having repeated id in objects,

  let data = new Array({id: 1},{id: 2},{id: 3},{id: 1},{id: 3});
  let unique = [];
  let tempArr = [];
  console.log('before', data);
  data.forEach((value, index) => {
    if (unique.indexOf(value.id) === -1) {
      unique.push(value.id);
    } else {
      tempArr.push(index);    
    }
  });
  tempArr.reverse();
  tempArr.forEach(ele => {
    data.splice(ele, 1);
  });
  console.log(data);

The easiest way is to transform values into strings to filter also nested objects values.

const uniq = (arg = []) => {
  const stringifyedArg = arg.map(value => JSON.stringify(value))
  return arg.filter((value, index, self) => {
    if (typeof value === 'object')
      return stringifyedArg.indexOf(JSON.stringify(value)) === index
    return self.indexOf(value) === index
  })
}

    console.log(uniq([21, 'twenty one', 21])) // [21, 'twenty one']
    console.log(uniq([{ a: 21 }, { a: 'twenty one' }, { a: 21 }])) // [{a: 21}, {a: 'twenty one'}]

For my part this was the easiest solution

// A way to check if the arrays are equal
const a = ['A', 'B', 'C'].sort().toString()
const b = ['A', 'C', 'B'].sort().toString()

console.log(a === b); // true


// Test Case
const data = [
  { group: 'A', name: 'SD' },
  { group: 'B', name: 'FI' },
  { group: 'A', name: 'SD' },
  { group: 'B', name: 'CO' }
];

// Return a new Array without dublocates
function unique(data) {
  return data.reduce(function (accumulator, currentValue) {
    // Convert to string in order to check if they are the same value.
    const currentKeys = Object.keys(currentValue).sort().toString();
    const currentValues = Object.values(currentValue).sort().toString();

    let hasObject = false
    
    for (const obj of accumulator) {
      // Convert keys and values into strings so we can
      // see if they are equal with the currentValue
      const keys = Object.keys(obj).sort().toString();
      const values = Object.values(obj).sort().toString();
      // Check if keys and values are equal
      if (keys === currentKeys && values === currentValues) {
        hasObject = true
      }
    }

    // Push the object if it does not exist already.
    if (!hasObject) {
      accumulator.push(currentValue)
    }

    return accumulator
  }, []);
}

// Run Test Case
console.log(unique(data)); // [ { group: 'A', name: 'SD' }, { group: 'B', name: 'FI' }, { group: 'B', name: 'CO' } ]

Here is another approach using comparators (I care more about clean code than performance):

const list = [
    {name: "Meier"},
    {name: "Hans"},
    {name: "Meier"},
]
const compare = (a, b) => a.name.localeCompare(b.name);
const uniqueNames = list.makeUnique(compare);
uniqueNames.pushIfAbsent({name: "Hans"}, compare);

Prototype declaration:

declare global {
    interface Array<T>  {
        pushIfAbsent(item: T, compare:(a:T, b:T)=>number): number;
    }
    interface Array<T>  {
        makeUnique(compare:(a:T, b:T)=>number): Array<T>;
    }
}
Array.prototype.pushIfAbsent = function <T>(this:T[], item:T, compare:(a:T, b:T)=>number) {
    if (!this.find(existing => compare(existing, item)===0)) {
        return this.push(item)
    } else {
        return this.length;
    }
}
Array.prototype.makeUnique = function <T>(this:T[], compare:(a:T, b:T)=>number) {
    return this.filter((existing, index, self) => self.findIndex(item => compare(existing, item) == 0) == index);
}

A modern approach that's extensible, fast, efficient and easy to read, using iter-ops library:

import {pipe, distinct} from 'iter-ops';

const input = [1, 1, 2, 2, 2, 3]; // our data

const i = pipe(input, distinct()); // distinct iterable

console.log([...i]); //=> [1, 2, 3]

And if your input is an array of objects, you will just provide a key selector for the distinct operator.

There's already bunch of great answers. Here's my approach.

var removeDuplicates = function(nums) {
    let filteredArr = [];
    nums.forEach((item) => {
        if(!filteredArr.includes(item)) {
            filteredArr.push(item);
        }
    })

  return filteredArr;
}

ES2016 .includes() One Method Simple Answer:

var arr = [1,5,2,4,1,6]
function getOrigs(arr) {
  let unique = []
  arr && arr.forEach(number => {
    !unique.includes(number) && unique.push(number)
    if (number === arr[arr.length - 1]) {
      console.log('unique: ', unique)
    }
  })
}
getOrigs(arr)

Use this instead:

• later ES version
• Simple question shouldn't use multiple advanced JS methods and push(), length() and forEach() are common
• Easier readability utilizing a closure
• Seems better on memory, garbage collection, and performance than the others
• Less lines of code: if you separated lines based on where there is a line ending, you would only need one line of logic (so you can call or refactor this one-liner however you want):

var arr = [1,5,2,4,1,6];
function getOrigs(arr) {let unique = []; 
  arr && arr.forEach(number => !unique.includes(number) && unique.push(number) && ((number === arr[arr.length - 1]) && console.log('unique: ', unique)))};
getOrigs(arr);

Try doing this:

let d_array = [1, 2, 2, 3, 'a', 'b', 'b', 'c'];
d_array = d_array.filter((x,i)=>d_array.indexOf(x)===i);
console.log(d_array); // [1, 2, 3, "a", "b", "c"]

This loops through the array, checks if the first found result of the same entry in the array is the current index, and if so, allows it to be in the array.

I wanted to remove duplicates from an array of objects. Duplicates had the same ids. Here is what I did.

// prev data
const prev = [
  {
    id: 1,
    name: "foo",
  },
  {
    id: 2,
    name: "baz",
  },
  {
    id: 1,
    name: "foo",
  },
];

// method:
// Step 1: put them in an object with the id as the key. Value of same id would get overriden.
// Step 2: get all the values.

const tempObj = {};
prev.forEach((n) => (tempObj[n.id] = n));
const next = Object.values(tempObj);

// result
[
  {
    id: 1,
    name: "foo",
  },
  {
    id: 2,
    name: "baz",
  }
];

Simple

const listWithDupes = [1, 2, 3, 4, 5, 6, 7, 7, 8, 8, 8, 8, 8, 8, 9];

const uniqueList = Object.values(listWithDupes.reduce((acc, next) => ({ ...acc,
  [next]: next
}), {}));

console.log(uniqueList);

An old way With 2 Loops.

let a = ["Messi", 23, false, 11, 17, true, 23, "Messi", "Sachin", "2023", true, "Sachin"];

let b = [];
    for (let i in a) {
        let found = false;
        for (let j in b) {
            if (a[i] === b[j]) {
                found = true;
                break;
            }
        }
        if (found === false) {
            b.push(a[i]);
        }
    }

console.log(b); //['Messi', 23, false, 11, 17, true, 'Sachin', '2023']

Yet another answer, just because I wrote one for my specific use case. I happened to be sorting the array anyway, and given I'm sorting I can use that to deduplicate.

Note that my sort deals with my specific data types, you might need a different sort depending on what sort of elements you have.

var sortAndDedup = function(array) {
  array.sort(function(a,b){
    if(isNaN(a) && isNaN(b)) { return a > b ? 1 : (a < b ? -1 : 0); }
    if(isNaN(a)) { return 1; }
    if(isNaN(b)) { return -1; }
    return a-b;
  });

  var newArray = [];
  var len = array.length;
  for(var i=0; i<len; i++){
    if(i === 0 || array[i] != array[i-1]){
      newArray.push(array[i]);
    }
  }
};

This script modify the array, filtering out duplicated values. It works with numbers and strings.

https://jsfiddle.net/qsdL6y5j/1/

    Array.prototype.getUnique = function () {
        var unique = this.filter(function (elem, pos) {
            return this.indexOf(elem) == pos;
        }.bind(this));
        this.length = 0;
        this.splice(0, 0, unique);
    }

    var duplicates = [0, 0, 1, 1, 2, 3, 1, 1, 0, 4, 4];
    duplicates.getUnique();
    alert(duplicates);

This version instead, allow you to return a new array with unique value keeping the original (just pass true).

https://jsfiddle.net/dj7qxyL7/

    Array.prototype.getUnique = function (createArray) {
        createArray = createArray === true ? true : false;
        var temp = JSON.stringify(this);
        temp = JSON.parse(temp);
        if (createArray) {
            var unique = temp.filter(function (elem, pos) {
                return temp.indexOf(elem) == pos;
            }.bind(this));
            return unique;
        }
        else {
            var unique = this.filter(function (elem, pos) {
                return this.indexOf(elem) == pos;
            }.bind(this));
            this.length = 0;
            this.splice(0, 0, unique);
        }
    }

    var duplicates = [0, 0, 1, 1, 2, 3, 1, 1, 0, 4, 4];
    console.log('++++ ovveride')
    duplicates.getUnique();
    console.log(duplicates);
    console.log('++++ new array')
    var duplicates2 = [0, 0, 1, 1, 2, 3, 1, 1, 0, 4, 4];
    var unique = duplicates2.getUnique(true);
    console.log(unique);
    console.log('++++ original')
    console.log(duplicates2);

Browser support:

Feature Chrome  Firefox (Gecko)     Internet Explorer   Opera   Safari
Basic support   (Yes)   1.5 (1.8)   9                   (Yes)   (Yes)

You can try this:

function removeDuplicates(arr){
  var temp = arr.sort();
  for(i = 0; i < temp.length; i++){
    if(temp[i] == temp[i + 1]){
      temp.splice(i,1);
      i--;
    }
  }
  return temp;
}

Using mongoose I had an array of ObjectIds to work with.

I had a array/list of Object Ids to work with which first needed to be set to an string and after the unique set, amended back to Object Ids.

var mongoose = require('mongoose')

var ids = [ObjectId("1"), ObjectId("2"), ObjectId("3")]

var toStringIds = ids.map(e => '' + e)
let uniqueIds = [...new Set(toStringIds)]
uniqueIds = uniqueIds.map(b => mongoose.Types.ObjectId(b))


console.log("uniqueIds :", uniqueIds)