Support repo license (#24872)

Close #278
Close #24076

## Solutions:
- Use
[google/licenseclassifier](https://github.com/google/licenseclassifier/)
Test result between
[google/licensecheck](https://github.com/google/licensecheck) and
[go-license-detector](https://github.com/go-enry/go-license-detector):
https://github.com/go-gitea/gitea/pull/24872#issuecomment-1560361167
Test result between
[google/licensecheck](https://github.com/google/licensecheck) and
[google/licenseclassifier](https://github.com/google/licenseclassifier/):
https://github.com/go-gitea/gitea/pull/24872#issuecomment-1576092178
- Generate License Convert Name List to avoid import license templates
with same contents
Gitea automatically get latest license data from[
spdx/license-list-data](https://github.com/spdx/license-list-data).
But unfortunately, some license templates have same contents. #20915
[click here to see the
list](https://github.com/go-gitea/gitea/pull/24872#issuecomment-1584141684)
So we will generate a list of these license templates with same contents
and create a new file to save the result when using `make
generate-license`. (Need to decide the save path)
- Save License info into a new table `repo_license`
Can easily support searching repo by license in the future.

## Screen shot
Single License:

![image](41260bd7-0b4c-4038-8592-508706cffa9f)

Multiple Licenses:

![image](34ce2f73-7e18-446b-9b96-ecc4fb61bd70)

Triggers:
- [x] Push commit to default branch
- [x] Create repo
- [x] Mirror repo
- [x] When Default Branch is changed, licenses should be updated

Todo:
- [x] Save Licenses info in to DB when there's a change to license file
in the commit
- [x] DB Migration
- [x] A nominal test?
- [x] Select which library to
use(https://github.com/go-gitea/gitea/pull/24872#issuecomment-1560361167)
- [x] API Support
- [x] Add repo license table
- ~Select license in settings if there are several licenses(Not
recommended)~
- License board(later, not in this PR)

![image](2c3c3bf8-bcc2-4c6d-8ce0-81d1a9733878)

---------

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: Denys Konovalov <kontakt@denyskon.de>
Co-authored-by: delvh <dev.lh@web.de>
Co-authored-by: KN4CK3R <admin@oldschoolhack.me>
Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: 6543 <m.huber@kithara.com>
Co-authored-by: a1012112796 <1012112796@qq.com>
Co-authored-by: techknowlogick <techknowlogick@gitea.com>

build/generate-licenses.go

@@ -1,3 +1,6 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
 //go:build ignore
 
 package main
@@ -5,6 +8,8 @@ package main
 import (
 	"archive/tar"
 	"compress/gzip"
+	"crypto/md5"
+	"encoding/hex"
 	"flag"
 	"fmt"
 	"io"
@@ -15,6 +20,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"code.gitea.io/gitea/build/license"
+	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/util"
 )
 
@@ -77,7 +84,7 @@ func main() {
 	}
 
 	tr := tar.NewReader(gz)
-
+	aliasesFiles := make(map[string][]string)
 	for {
 		hdr, err := tr.Next()
 
@@ -97,26 +104,73 @@ func main() {
 			continue
 		}
 
-		if strings.HasPrefix(filepath.Base(hdr.Name), "README") {
+		fileBaseName := filepath.Base(hdr.Name)
+		licenseName := strings.TrimSuffix(fileBaseName, ".txt")
+
+		if strings.HasPrefix(fileBaseName, "README") {
 			continue
 		}
 
-		if strings.HasPrefix(filepath.Base(hdr.Name), "deprecated_") {
+		if strings.HasPrefix(fileBaseName, "deprecated_") {
 			continue
 		}
-		out, err := os.Create(path.Join(destination, strings.TrimSuffix(filepath.Base(hdr.Name), ".txt")))
+		out, err := os.Create(path.Join(destination, licenseName))
 		if err != nil {
 			log.Fatalf("Failed to create new file. %s", err)
 		}
 
 		defer out.Close()
 
-		if _, err := io.Copy(out, tr); err != nil {
+		// some license files have same content, so we need to detect these files and create a convert map into a json file
+		// Later we use this convert map to avoid adding same license content with different license name
+		h := md5.New()
+		// calculate md5 and write file in the same time
+		r := io.TeeReader(tr, h)
+		if _, err := io.Copy(out, r); err != nil {
 			log.Fatalf("Failed to write new file. %s", err)
 		} else {
 			fmt.Printf("Written %s\n", out.Name())
+
+			md5 := hex.EncodeToString(h.Sum(nil))
+			aliasesFiles[md5] = append(aliasesFiles[md5], licenseName)
 		}
 	}
 
+	// generate convert license name map
+	licenseAliases := make(map[string]string)
+	for _, fileNames := range aliasesFiles {
+		if len(fileNames) > 1 {
+			licenseName := license.GetLicenseNameFromAliases(fileNames)
+			if licenseName == "" {
+				// license name should not be empty as expected
+				// if it is empty, we need to rewrite the logic of GetLicenseNameFromAliases
+				log.Fatalf("GetLicenseNameFromAliases: license name is empty")
+			}
+			for _, fileName := range fileNames {
+				licenseAliases[fileName] = licenseName
+			}
+		}
+	}
+	// save convert license name map to file
+	b, err := json.Marshal(licenseAliases)
+	if err != nil {
+		log.Fatalf("Failed to create json bytes. %s", err)
+	}
+
+	licenseAliasesDestination := filepath.Join(destination, "etc", "license-aliases.json")
+	if err := os.MkdirAll(filepath.Dir(licenseAliasesDestination), 0o755); err != nil {
+		log.Fatalf("Failed to create directory for license aliases json file. %s", err)
+	}
+
+	f, err := os.Create(licenseAliasesDestination)
+	if err != nil {
+		log.Fatalf("Failed to create license aliases json file. %s", err)
+	}
+	defer f.Close()
+
+	if _, err = f.Write(b); err != nil {
+		log.Fatalf("Failed to write license aliases json file. %s", err)
+	}
+
 	fmt.Println("Done")
 }

build/license/aliasgenerator.go

@@ -0,0 +1,41 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package license
+
+import "strings"
+
+func GetLicenseNameFromAliases(fnl []string) string {
+	if len(fnl) == 0 {
+		return ""
+	}
+
+	shortestItem := func(list []string) string {
+		s := list[0]
+		for _, l := range list[1:] {
+			if len(l) < len(s) {
+				s = l
+			}
+		}
+		return s
+	}
+	allHasPrefix := func(list []string, s string) bool {
+		for _, l := range list {
+			if !strings.HasPrefix(l, s) {
+				return false
+			}
+		}
+		return true
+	}
+
+	sl := shortestItem(fnl)
+	slv := strings.Split(sl, "-")
+	var result string
+	for i := len(slv); i >= 0; i-- {
+		result = strings.Join(slv[:i], "-")
+		if allHasPrefix(fnl, result) {
+			return result
+		}
+	}
+	return ""
+}

build/license/aliasgenerator_test.go

@@ -0,0 +1,39 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package license
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetLicenseNameFromAliases(t *testing.T) {
+	tests := []struct {
+		target string
+		inputs []string
+	}{
+		{
+			// real case which you can find in license-aliases.json
+			target: "AGPL-1.0",
+			inputs: []string{
+				"AGPL-1.0-only",
+				"AGPL-1.0-or-late",
+			},
+		},
+		{
+			target: "",
+			inputs: []string{
+				"APSL-1.0",
+				"AGPL-1.0-only",
+				"AGPL-1.0-or-late",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		result := GetLicenseNameFromAliases(tt.inputs)
+		assert.Equal(t, result, tt.target)
+	}
+}