Shaman: use relative paths for symlinks #104332

Merged
Sybren A. Stüvel merged 1 commits from dr.sybren/flamenco:shaman-relative-symlinks into main 2024-09-05 12:12:50 +02:00
3 changed files with 64 additions and 23 deletions

View File

@ -44,18 +44,31 @@ func TestCheckout(t *testing.T) {
assert.FileExists(t, filepath.Join(coPath, "httpstuff.py")) assert.FileExists(t, filepath.Join(coPath, "httpstuff.py"))
assert.FileExists(t, filepath.Join(coPath, "много ликова.py")) assert.FileExists(t, filepath.Join(coPath, "много ликова.py"))
storePath := manager.fileStore.StoragePath() storePath, err := filepath.Rel(
assertLinksTo(t, filepath.Join(coPath, "subdir", "replacer.py"), manager.checkoutBasePath,
filepath.Join(storePath, "59", "0c148428d5c35fab3ebad2f3365bb469ab9c531b60831f3e826c472027a0b9", "3367.blob")) manager.fileStore.StoragePath(),
assertLinksTo(t, filepath.Join(coPath, "feed.py"), )
filepath.Join(storePath, "80", "b749c27b2fef7255e7e7b3c2029b03b31299c75ff1f1c72732081c70a713a3", "7488.blob")) require.NoError(t, err)
assertLinksTo(t, filepath.Join(coPath, "httpstuff.py"), assertLinksTo(t,
filepath.Join(storePath, "91", "4853599dd2c351ab7b82b219aae6e527e51518a667f0ff32244b0c94c75688", "486.blob")) // Two extra '..' for 'á hausinn á þér/subdir'.
assertLinksTo(t, filepath.Join(coPath, "много ликова.py"), filepath.Join("..", "..", storePath, "59", "0c148428d5c35fab3ebad2f3365bb469ab9c531b60831f3e826c472027a0b9", "3367.blob"),
filepath.Join(storePath, "d6", "fc7289b5196cc96748ea72f882a22c39b8833b457fe854ef4c03a01f5db0d3", "7217.blob")) filepath.Join(coPath, "subdir", "replacer.py"),
)
assertLinksTo(t,
filepath.Join("..", storePath, "80", "b749c27b2fef7255e7e7b3c2029b03b31299c75ff1f1c72732081c70a713a3", "7488.blob"),
filepath.Join(coPath, "feed.py"),
)
assertLinksTo(t,
filepath.Join("..", storePath, "91", "4853599dd2c351ab7b82b219aae6e527e51518a667f0ff32244b0c94c75688", "486.blob"),
filepath.Join(coPath, "httpstuff.py"),
)
assertLinksTo(t,
filepath.Join("..", storePath, "d6", "fc7289b5196cc96748ea72f882a22c39b8833b457fe854ef4c03a01f5db0d3", "7217.blob"),
filepath.Join(coPath, "много ликова.py"),
)
} }
func assertLinksTo(t *testing.T, linkPath, expectedTarget string) { func assertLinksTo(t *testing.T, expectedTarget, linkPath string) {
actualTarget, err := os.Readlink(linkPath) actualTarget, err := os.Readlink(linkPath)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, expectedTarget, actualTarget) assert.Equal(t, expectedTarget, actualTarget)

View File

@ -202,22 +202,46 @@ func (m *Manager) EraseCheckout(checkoutID string) error {
// It does *not* do any validation of the validity of the paths! // It does *not* do any validation of the validity of the paths!
func (m *Manager) SymlinkToCheckout(blobPath, checkoutPath, symlinkRelativePath string) error { func (m *Manager) SymlinkToCheckout(blobPath, checkoutPath, symlinkRelativePath string) error {
symlinkPath := filepath.Join(checkoutPath, symlinkRelativePath) symlinkPath := filepath.Join(checkoutPath, symlinkRelativePath)
logger := log.With(). logger := log.With().Str("symlinkPath", symlinkPath).Logger()
Str("blobPath", blobPath).
Str("symlinkPath", symlinkPath).
Logger()
blobPath, err := filepath.Abs(blobPath) blobAbsolute, err := filepath.Abs(blobPath)
if err != nil { if err != nil {
logger.Error().Err(err).Msg("shaman: unable to make blobPath absolute") logger.Error().
Str("blobPath", blobPath).
Err(err).Msg("shaman: unable to make blobPath absolute")
return err return err
} }
if blobAbsolute != blobPath {
logger.Trace().
Str("input", blobPath).
Str("absolute", blobAbsolute).
Msg("shaman: made blobpath absolute")
}
// Try and make blobAbsolute relative to the symlink target directory, so that
// mounting the symlinked storage at a different prefix works as well.
symlinkDir := filepath.Dir(symlinkPath)
blobRelativeToCheckout, err := filepath.Rel(symlinkDir, blobAbsolute)
if err != nil {
logger.Warn().
Str("blobPath", blobAbsolute).
AnErr("cause", err).
Msg("shaman: unable to make blobPath relative, will use absolute path")
blobRelativeToCheckout = blobAbsolute
}
logger.Trace().
Str("absolute", blobAbsolute).
Str("relative", blobRelativeToCheckout).
Str("symlinkDir", symlinkDir).
Msg("shaman: made blobpath relative")
logger = logger.With().Str("blobPath", blobRelativeToCheckout).Logger()
logger.Debug().Msg("shaman: creating symlink") logger.Debug().Msg("shaman: creating symlink")
// This is expected to fail sometimes, because we don't create parent directories yet. // This is expected to fail sometimes, because we don't create parent directories yet.
// We only create those when we get a failure from symlinking. // We only create those when we get a failure from symlinking.
err = os.Symlink(blobPath, symlinkPath) err = os.Symlink(blobRelativeToCheckout, symlinkPath)
switch { switch {
case err == nil: case err == nil:
return nil return nil
@ -232,7 +256,7 @@ func (m *Manager) SymlinkToCheckout(blobPath, checkoutPath, symlinkRelativePath
Msg("shaman: unable to create symlink as it already exists, but also it cannot be read") Msg("shaman: unable to create symlink as it already exists, but also it cannot be read")
return err return err
} }
if linkTarget != blobPath { if linkTarget != blobRelativeToCheckout {
logger.Error(). logger.Error().
AnErr("symlinkError", err). AnErr("symlinkError", err).
Str("alreadyLinkedFrom", linkTarget). Str("alreadyLinkedFrom", linkTarget).
@ -251,7 +275,7 @@ func (m *Manager) SymlinkToCheckout(blobPath, checkoutPath, symlinkRelativePath
logger.Error().Err(err).Msg("shaman: unable to create parent directory") logger.Error().Err(err).Msg("shaman: unable to create parent directory")
return err return err
} }
if err := os.Symlink(blobPath, symlinkPath); err != nil { if err := os.Symlink(blobRelativeToCheckout, symlinkPath); err != nil {
logger.Error().Err(err).Msg("shaman: unable to create symlink, after creating parent directory") logger.Error().Err(err).Msg("shaman: unable to create symlink, after creating parent directory")
return err return err
} }
@ -263,7 +287,7 @@ func (m *Manager) SymlinkToCheckout(blobPath, checkoutPath, symlinkRelativePath
// Change the modification time of the blob to mark it as 'referenced' just now. // Change the modification time of the blob to mark it as 'referenced' just now.
m.wg.Add(1) m.wg.Add(1)
go func() { go func() {
if err := touchFile(blobPath); err != nil { if err := touchFile(blobAbsolute); err != nil {
logger.Warn().Err(err).Msg("shaman: unable to touch blob path") logger.Warn().Err(err).Msg("shaman: unable to touch blob path")
} }
m.wg.Done() m.wg.Done()

View File

@ -59,7 +59,7 @@ func TestSymlinkToCheckout(t *testing.T) {
defer cleanup() defer cleanup()
// Fake an older file. // Fake an older file.
blobPath := filepath.Join(manager.checkoutBasePath, "jemoeder.blob") blobPath := filepath.Join(manager.fileStore.StoragePath(), "opjehoofd.blob")
err := os.WriteFile(blobPath, []byte("op je hoofd"), 0600) err := os.WriteFile(blobPath, []byte("op je hoofd"), 0600)
require.NoError(t, err) require.NoError(t, err)
@ -67,14 +67,14 @@ func TestSymlinkToCheckout(t *testing.T) {
err = os.Chtimes(blobPath, wayBackWhen, wayBackWhen) err = os.Chtimes(blobPath, wayBackWhen, wayBackWhen)
require.NoError(t, err) require.NoError(t, err)
symlinkRelativePath := "path/to/jemoeder.txt" symlinkRelativePath := "path/to/opjehoofd.txt"
err = manager.SymlinkToCheckout(blobPath, manager.checkoutBasePath, symlinkRelativePath) err = manager.SymlinkToCheckout(blobPath, manager.checkoutBasePath, symlinkRelativePath)
require.NoError(t, err) require.NoError(t, err)
err = manager.SymlinkToCheckout(blobPath, manager.checkoutBasePath, symlinkRelativePath) err = manager.SymlinkToCheckout(blobPath, manager.checkoutBasePath, symlinkRelativePath)
require.NoError(t, err, "symlinking a file twice should not be an issue") require.NoError(t, err, "symlinking a file twice should not be an issue")
// Wait for touch() calls to be done. // Wait for the manager to be done updating mtimes.
manager.wg.Wait() manager.wg.Wait()
// The blob should have been touched to indicate it was referenced just now. // The blob should have been touched to indicate it was referenced just now.
@ -89,6 +89,10 @@ func TestSymlinkToCheckout(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
assert.True(t, stat.Mode()&os.ModeType == os.ModeSymlink, assert.True(t, stat.Mode()&os.ModeType == os.ModeSymlink,
"%v should be a symlink", symlinkPath) "%v should be a symlink", symlinkPath)
contents, err := os.ReadFile(symlinkPath)
require.NoError(t, err)
assert.Equal(t, string(contents), "op je hoofd")
} }
func TestPrepareCheckout(t *testing.T) { func TestPrepareCheckout(t *testing.T) {