Vulkan shadow map artifacts

I am experiencing some weird shadow artifacts.

I suspect it to be some sort of self shadowing/depth bias thing.

My shader for the shadow map is quite simple:

shadow_map.vert

#version 450

layout(location = 0) in vec3 in_position;

layout(binding = 0) uniform uniform_scene {
  mat4 light_space;
} scene;

layout(push_constant) uniform uniform_object {
  mat4 model;
} object;

void main() {
  gl_Position = scene.light_space * object.model * vec4(in_position, 1.0);
}

shadow_map.frag:

#version 450

// layout(location = 0) out float out_depth;

void main() {
  // out_depth = gl_FragCoord.z;
}

And for actually doing the depth test for shadows I followed along learnopengl.com since I thought the shader code should be the same for vulkan and opengl.

base.vert

#version 450

struct per_mesh_data {
  mat4 model;
  mat4 normal;
  vec4 tint;
}; // struct per_mesh_data

layout(location = 0) in vec3 in_position;
layout(location = 1) in vec3 in_normal;
layout(location = 2) in vec2 in_uv;

layout(location = 0) out vec3 out_position;
layout(location = 1) out vec3 out_normal;
layout(location = 2) out vec2 out_uv;
layout(location = 3) out vec4 out_light_space_position;
layout(location = 4) out vec4 out_tint;

layout(binding = 0) uniform uniform_scene {
  mat4 view;
  mat4 projection;
  vec3 camera_position;
  mat4 light_space;
  vec3 light_direction;
  vec4 light_color;
} scene;

layout(binding = 1) buffer buffer_mesh_data {
  per_mesh_data data[];
} mesh_data;

const mat4 depth_bias = mat4( 
    0.5, 0.0, 0.0, 0.0,
    0.0, 0.5, 0.0, 0.0,
    0.0, 0.0, 1.0, 0.0,
    0.5, 0.5, 0.0, 1.0
);

void main() {
  per_mesh_data data = mesh_data.data[gl_InstanceIndex]; 

  out_position = vec3(data.model * vec4(in_position, 1.0));
  out_normal = normalize(mat3(data.normal) * in_normal);
  out_uv = in_uv;
  out_light_space_position = (depth_bias * scene.light_space) * vec4(out_position, 1.0);
  out_tint = data.tint;

  gl_Position = scene.projection * scene.view * vec4(out_position, 1.0);
}

base.frag

#version 450

#include "../common/lighting.glsl"
#include "../common/material.glsl"

layout(location = 0) in vec3 in_position;
layout(location = 1) in vec3 in_normal;
layout(location = 2) in vec2 in_uv;
layout(location = 3) in vec4 in_light_space_position;
layout(location = 4) in vec4 in_tint;

layout(location = 0) out vec4 out_color;

layout(binding = 0) uniform uniform_scene {
  mat4 view;
  mat4 projection;
  vec3 camera_position;
  mat4 light_space;
  vec3 light_direction;
  vec4 light_color;
} scene;

layout(binding = 2) uniform sampler2D image;
layout(binding = 3) uniform sampler2D shadow_map;

layout(push_constant) uniform uniform_object {
  mat4 model;
  mat4 normal;
} object;

const material default_material = material(
  vec4(1.0, 1.0, 1.0, 1.0),
  vec4(1.0, 1.0, 1.0, 1.0),
  vec4(0.5, 0.5, 0.5, 1.0),
  16.0
);

vec4 phong_lighting(vec3 light_direction, vec3 view_direction, vec3 normal, material material) {
  // Ambient
  vec4 ambient_color = scene.light_color * material.ambient;

  // Diffuse
  float diffuse_factor = max(dot(light_direction, normal), 0.0);
  vec4 diffuse_color = diffuse_factor * scene.light_color * material.diffuse;

  // Specular
  vec3 halfway_direction = normalize(light_direction + view_direction);  
  float specular_factor = pow(max(dot(normal, halfway_direction), 0.0), material.shininess);
  vec4 specular_color = specular_factor * scene.light_color * material.specular; 

  return ambient_color + diffuse_color + specular_color;
}

float pcf_shadow(vec3 light_direction) {
  vec2 texture_size = textureSize(shadow_map, 0);
  vec2 texel_size = 1.0 / texture_size;

  vec3 coordinates = in_light_space_position.xyz / in_light_space_position.w;

  if (coordinates.z > 1.0 || coordinates.z < -1.0) {
    return 0.0;
  }

  float shadow = 0.0;

  float bias = max(0.001 * (1.0 - dot(in_normal, light_direction)), 0.0001);
  // float bias = 0.001;
  
  float current_depth = coordinates.z;
  
  int count = 0;
  int range = 2;

  for (int x = -range; x <= range; ++x) {
    for (int y = -range; y <= range; ++y) {
      float pcf_depth = texture(shadow_map, coordinates.xy + vec2(x, y) * texel_size).r;
      shadow += (current_depth - bias) > pcf_depth ? 1.0 : 0.0;
      ++count;
    }
  }

  return shadow / float(count);
}

void main() {
  vec3 light_direction = normalize(-scene.light_direction);
  vec3 view_direction = normalize(scene.camera_position - in_position);

  // Calculate lighting
  vec4 lighting = phong_lighting(light_direction, view_direction, in_normal, default_material);

  // Calculate shadow
  float shadow_factor = pcf_shadow(light_direction);

  // Sample texture
  vec4 sampled_color = texture(image, in_uv);

  vec4 shaded_lighting = mix(lighting * (1.0 - shadow_factor), lighting * 0.2, shadow_factor);

  out_color = shaded_lighting * sampled_color * in_tint;
}

The included files here just define structs like material and directional_light